diff options
Diffstat (limited to 'lib/gitlab')
300 files changed, 16896 insertions, 2977 deletions
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 0f0ecd82a32..1920e1443da 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -21,6 +21,7 @@ module Gitlab :related_class, :feature_category, :artifact_size, + :artifact_used_cdn, :artifacts_dependencies_size, :artifacts_dependencies_count, :root_caller_id @@ -38,6 +39,7 @@ module Gitlab Attribute.new(:related_class, String), Attribute.new(:feature_category, String), Attribute.new(:artifact, ::Ci::JobArtifact), + Attribute.new(:artifact_used_cdn, Object), Attribute.new(:artifacts_dependencies_size, Integer), Attribute.new(:artifacts_dependencies_count, Integer), Attribute.new(:root_caller_id, String) @@ -91,6 +93,7 @@ module Gitlab assign_hash_if_value(hash, :remote_ip) assign_hash_if_value(hash, :related_class) assign_hash_if_value(hash, :feature_category) + assign_hash_if_value(hash, :artifact_used_cdn) assign_hash_if_value(hash, :artifacts_dependencies_size) assign_hash_if_value(hash, :artifacts_dependencies_count) diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6213dd203c4..c567df8e133 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -221,6 +221,8 @@ module Gitlab end if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? + ::PersonalAccessTokens::LastUsedService.new(token).execute + Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 1fed2b263da..26be7c8aa60 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -217,11 +217,7 @@ module Gitlab def build_new_user(skip_confirmation: true) user_params = user_attributes.merge(skip_confirmation: skip_confirmation) - new_user = Users::AuthorizedBuildService.new(nil, user_params).execute - - persist_accepted_terms_if_required(new_user) - - new_user + Users::AuthorizedBuildService.new(nil, user_params).execute end def user_attributes @@ -249,15 +245,6 @@ module Gitlab } end - def persist_accepted_terms_if_required(new_user) - if Feature.enabled?(:update_oauth_registration_flow) && - Gitlab::CurrentSettings.current_application_settings.enforce_terms? - - terms = ApplicationSetting::Term.latest - Users::RespondToTermsService.new(new_user, terms).execute(accepted: true) - end - end - def sync_profile_from_provider? Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) end diff --git a/lib/gitlab/auth_logger.rb b/lib/gitlab/auth_logger.rb index 6d3edba02b0..763430df335 100644 --- a/lib/gitlab/auth_logger.rb +++ b/lib/gitlab/auth_logger.rb @@ -7,3 +7,5 @@ module Gitlab end end end + +Gitlab::AuthLogger.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb new file mode 100644 index 00000000000..bd61d1a0f07 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +Gitlab::BackgroundMigration::BackfillEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb new file mode 100644 index 00000000000..300f2cff6ca --- /dev/null +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This syncs the data to `internal` from `confidential` as we rename the column. + class BackfillInternalOnNotes < BatchedMigrationJob + scope_to -> (relation) { relation.where(confidential: true) } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + sub_batch.update_all(internal: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb new file mode 100644 index 00000000000..b8a51b576b6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill namespace_details for a range of namespaces + class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + upsert_namespace_details(sub_batch) + end + end + + def upsert_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT namespaces.description, namespaces.description_html, namespaces.cached_markdown_version, now(), now(), namespaces.id + FROM namespaces + WHERE namespaces.id IN(#{relation.select(:id).to_sql}) + AND namespaces.type <> 'Project' + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb new file mode 100644 index 00000000000..c3e1019b72f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned operational vulnerabilities (without findings) + class DeleteOrphanedOperationalVulnerabilities < ::Gitlab::BackgroundMigration::BatchedMigrationJob + REPORT_TYPES = { + cluster_image_scanning: 7, + custom: 99 + }.freeze + + NOT_EXISTS_SQL = <<-SQL + NOT EXISTS ( + SELECT FROM vulnerability_occurrences + WHERE "vulnerability_occurrences"."vulnerability_id" = "vulnerabilities"."id" + ) + SQL + + scope_to ->(relation) do + relation + .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) + end + + def perform + each_sub_batch(operation_name: :delete_orphaned_operational_vulnerabilities) do |sub_batch| + sub_batch + .where(NOT_EXISTS_SQL) + .delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb new file mode 100644 index 00000000000..7d78795bea9 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(member_namespace_id: nil) } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + deleted_members_data = sub_batch.map do |m| + { id: m.id, source_id: m.source_id, source_type: m.source_type } + end + + deleted_count = sub_batch.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid member records', + deleted_count: deleted_count, + deleted_member_data: deleted_members_data }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..2257dc016be --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalMergeRequestRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalMergeRequestRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb new file mode 100644 index 00000000000..1d0c0010551 --- /dev/null +++ b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete merge request level rules + # as this feature exists only in EE + class PopulateApprovalProjectRulesWithSecurityOrchestration < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration.prepend_mod_with('Gitlab::BackgroundMigration::PopulateApprovalProjectRulesWithSecurityOrchestration') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb new file mode 100644 index 00000000000..952f3b0e3c3 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token_encrypted values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_encrypted_values) do |sub_batch| + # Reset duplicate runner encrypted tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token_encrypted: sub_batch.select(:token_encrypted).distinct) + .group(:token_encrypted) + .having('COUNT(*) > 1') + .pluck(:token_encrypted) + + return if duplicate_tokens.empty? + + batchable_model.where(token_encrypted: duplicate_tokens).update_all(token_encrypted: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb new file mode 100644 index 00000000000..cfd6a4e4091 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate token values in ci_runners table in batches + class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_values) do |sub_batch| + # Reset duplicate runner tokens that would prevent creating an unique index. + nullify_duplicate_ci_runner_token_values(sub_batch) + end + end + + private + + def nullify_duplicate_ci_runner_token_values(sub_batch) + batchable_model = define_batchable_model(batch_table, connection: connection) + + duplicate_tokens = batchable_model + .where(token: sub_batch.select(:token).distinct) + .group(:token) + .having('COUNT(*) > 1') + .pluck(:token) + + batchable_model.where(token: duplicate_tokens).update_all(token: nil) if duplicate_tokens.any? + end + end + end +end diff --git a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb new file mode 100644 index 00000000000..84183753158 --- /dev/null +++ b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The `ci_pipeline_artifacts.locked` column was added in + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97194 to + # speed up the finding of expired, pipeline artifacts. By default, + # the value is "unknown" (2), but the correct value should be the + # value of the associated `ci_pipelines.locked` value. This class + # does an UPDATE join to make the values match. + class UpdateCiPipelineArtifactsUnknownLockedStatus < BatchedMigrationJob + def perform + connection.exec_query(<<~SQL) + UPDATE ci_pipeline_artifacts + SET locked = ci_pipelines.locked + FROM ci_pipelines + WHERE ci_pipeline_artifacts.id BETWEEN #{start_id} AND #{end_id} + AND ci_pipeline_artifacts.locked = 2 + AND ci_pipelines.id = ci_pipeline_artifacts.pipeline_id; + SQL + end + end + end +end diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb deleted file mode 100644 index 103ce644f2b..00000000000 --- a/lib/gitlab/batch_pop_queueing.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - ## - # This class is a queuing system for processing expensive tasks in an atomic manner - # with batch poping to let you optimize the total processing time. - # - # In usual queuing system, the first item started being processed immediately - # and the following items wait until the next items have been popped from the queue. - # On the other hand, this queueing system, the former part is same, however, - # it pops the enqueued items as batch. This is especially useful when you want to - # drop redundant items from the queue in order to process important items only, - # thus it's more efficient than the traditional queueing system. - # - # Caveats: - # - The order of the items are not guaranteed because of `sadd` (Redis Sets). - # - # Example: - # ``` - # class TheWorker - # def perform - # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| - # item = extract_the_most_important_item_from(items_in_queue) - # expensive_process(item) - # end - # - # if result[:status] == :finished && result[:new_items].present? - # item = extract_the_most_important_item_from(items_in_queue) - # TheWorker.perform_async(item.id) - # end - # end - # end - # ``` - # - class BatchPopQueueing - attr_reader :namespace, :queue_id - - EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour - MAX_COUNTS_OF_POP_ALL = 1000 - - # Initialize queue - # - # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. - # @param [String] queue_id The identifier of the queue. - # @return [Boolean] - def initialize(namespace, queue_id) - raise ArgumentError if namespace.empty? || queue_id.empty? - - @namespace = namespace - @queue_id = queue_id - end - - ## - # Execute the given block in an exclusive lock. - # If there is the other thread has already working on the block, - # it enqueues the items without processing the block. - # - # @param [Array<String>] new_items New items to be added to the queue. - # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. - # @return [Hash] - # - status => One of the `:enqueued` or `:finished`. - # - new_items => Newly enqueued items during the given block had been processed. - # - # NOTE: If an exception is raised in the block, the poppped items will not be recovered. - # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. - def safe_execute(new_items, lock_timeout: 10.minutes, &block) - enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) - - lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) - - return { status: :enqueued } unless uuid = lease.try_obtain - - begin - all_args = pop_all - - yield all_args if block - - { status: :finished, new_items: peek_all } - ensure - Gitlab::ExclusiveLease.cancel(lock_key, uuid) - end - end - - private - - def lock_key - @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" - end - - def queue_key - @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" - end - - def enqueue(items, expire_time) - Gitlab::Redis::Queues.with do |redis| - redis.sadd(queue_key, items) - redis.expire(queue_key, expire_time.to_i) - end - end - - def pop_all - Gitlab::Redis::Queues.with do |redis| - redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) - end - end - - def peek_all - Gitlab::Redis::Queues.with do |redis| - redis.smembers(queue_key) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d58de7eb211..7de6be45349 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -67,6 +67,14 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def allocate_issues_internal_id!(project, client) + last_bitbucket_issue = client.last_issue(repo) + + return unless last_bitbucket_issue + + Issue.track_project_iid!(project, last_bitbucket_issue.iid) + end + def repo @repo ||= client.repo(project.import_source) end @@ -84,6 +92,10 @@ module Gitlab def import_issues return unless repo.issues_enabled? + # If a user creates an issue while the import is in progress, this can lead to an import failure. + # The workaround is to allocate IIDs before starting the importer. + allocate_issues_internal_id!(project, client) + create_labels issue_type_id = WorkItems::Type.default_issue_type.id diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 99ce1119c17..9209c9b4927 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -108,7 +108,7 @@ module Gitlab return self.loaded unless self.loaded.nil? Gitlab::Redis::Cache.with do |redis| - redis.exists(cache_key) + redis.exists?(cache_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 48b6ca59367..024fa48c066 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -126,7 +126,6 @@ module Gitlab end def increment_cache_metric(render_type:, total_count:, miss_count:) - return unless Feature.enabled?(:add_timing_to_certain_cache_actions) return unless caller_id metric_name = :cached_object_operations_total @@ -146,17 +145,13 @@ module Gitlab end def time_action(render_type:, &block) - if Feature.enabled?(:add_timing_to_certain_cache_actions) - real_start = Gitlab::Metrics::System.monotonic_time + real_start = Gitlab::Metrics::System.monotonic_time - presented_object = yield + presented_object = yield - real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) - presented_object - else - yield - end + presented_object end def real_duration_histogram(render_type) diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 1034f5eacef..4069a683ceb 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -41,7 +41,6 @@ module Gitlab def find_modified_paths(pipeline) return unless pipeline - return pipeline.modified_paths unless ::Feature.enabled?(:ci_rules_changes_compare, pipeline.project) compare_to_sha = find_compare_to_sha(pipeline) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 438fa1cb3b2..661c6fb87e3 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -85,6 +85,10 @@ module Gitlab root.workflow_entry.rules_value end + def workflow_name + root.workflow_entry.name + end + def normalized_jobs @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end diff --git a/lib/gitlab/ci/config/entry/current_variables.rb b/lib/gitlab/ci/config/entry/current_variables.rb deleted file mode 100644 index 3b6721ec92d..00000000000 --- a/lib/gitlab/ci/config/entry/current_variables.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents CI/CD variables. - # The class will be renamed to `Variables` when removing the FF `ci_variables_refactoring_to_variable`. - # - class CurrentVariables < ::Gitlab::Config::Entry::ComposableHash - include ::Gitlab::Config::Entry::Validatable - - validations do - validates :config, type: Hash - end - - # Enable these lines when removing the FF `ci_variables_refactoring_to_variable` - # and renaming this class to `Variables`. - # def self.default(**) - # {} - # end - - def value - @entries.to_h do |key, entry| - [key.to_s, entry.value] - end - end - - def value_with_data - @entries.to_h do |key, entry| - [key.to_s, entry.value_with_data] - end - end - - private - - def composable_class(_name, _config) - Entry::Variable - end - - def composable_metadata - { allowed_value_data: opt(:allowed_value_data) } - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/legacy_variables.rb b/lib/gitlab/ci/config/entry/legacy_variables.rb deleted file mode 100644 index 5379f707537..00000000000 --- a/lib/gitlab/ci/config/entry/legacy_variables.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - class Config - module Entry - ## - # Entry that represents environment variables. - # This is legacy implementation and will be removed with the FF `ci_variables_refactoring_to_variable`. - # - class LegacyVariables < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - ALLOWED_VALUE_DATA = %i[value description].freeze - - validations do - validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? - validates :config, variables: true, unless: :use_value_data? - end - - def value - @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } - end - - def value_with_data - @config.to_h { |key, value| [key.to_s, expand_value(value)] } - end - - def use_value_data? - opt(:use_value_data) - end - - private - - def expand_value(value) - if value.is_a?(Hash) - { value: value[:value].to_s, description: value[:description] }.compact - else - { value: value.to_s } - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 57e89bd7bc5..1d7d8617c74 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -48,10 +48,9 @@ module Gitlab description: 'Script that will be executed after each job.', reserved: true - # use_value_data will be removed with the FF ci_variables_refactoring_to_variable entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { use_value_data: true, allowed_value_data: %i[value description] }, + metadata: { allowed_value_data: %i[value description], allow_array_value: true }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/timeout.rb b/lib/gitlab/ci/config/entry/timeout.rb index 0bffa9340de..5769ea22b06 100644 --- a/lib/gitlab/ci/config/entry/timeout.rb +++ b/lib/gitlab/ci/config/entry/timeout.rb @@ -5,7 +5,7 @@ module Gitlab class Config module Entry ## - # Entry that represents the interrutible value. + # Entry that represents the interruptible value. # class Timeout < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 253888aadeb..54c153c8b07 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -10,6 +10,7 @@ module Gitlab class Variable < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } + strategy :ComplexArrayVariable, if: -> (config) { ComplexArrayVariable.applies_to?(config) } class SimpleVariable < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -39,7 +40,7 @@ module Gitlab class << self def applies_to?(config) - config.is_a?(Hash) + config.is_a?(Hash) && !config[:value].is_a?(Array) end end @@ -86,6 +87,34 @@ module Gitlab end end + class ComplexArrayVariable < ComplexVariable + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + config.is_a?(Hash) && config[:value].is_a?(Array) + end + end + + validations do + validates :config_value, array_of_strings: true, allow_nil: false, if: :config_value_defined? + + validate do + next if opt(:allow_array_value) + + errors.add(:config, 'value must be an alphanumeric string') + end + end + + def value + config_value.first + end + + def value_with_data + super.merge(value_options: config_value).compact + end + end + class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["variable definition must be either a string or a hash"] diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 0284958d9d4..4430a11dda7 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -6,20 +6,38 @@ module Gitlab module Entry ## # Entry that represents CI/CD variables. - # CurrentVariables will be renamed to this class when removing the FF `ci_variables_refactoring_to_variable`. - # - class Variables - def self.new(...) - if YamlProcessor::FeatureFlags.enabled?(:ci_variables_refactoring_to_variable) - CurrentVariables.new(...) - else - LegacyVariables.new(...) - end + class Variables < ::Gitlab::Config::Entry::ComposableHash + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Hash end def self.default(**) {} end + + def value + @entries.to_h do |key, entry| + [key.to_s, entry.value] + end + end + + def value_with_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_data] + end + end + + private + + def composable_class(_name, _config) + Entry::Variable + end + + def composable_metadata + { allowed_value_data: opt(:allowed_value_data), allow_array_value: opt(:allow_array_value) } + end end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 5bc992a38a0..691d9e2d48b 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -6,12 +6,17 @@ module Gitlab module Entry class Workflow < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[rules].freeze + ALLOWED_KEYS = %i[rules name].freeze + + attributes :name validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS + validates :name, allow_nil: true, length: { minimum: 1, maximum: 255 } end entry :rules, Entry::Rules, diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index ec628399785..138e79db331 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -10,7 +10,6 @@ module Gitlab TimeoutError = Class.new(StandardError) MAX_INCLUDES = 100 - TRIAL_MAX_INCLUDES = 250 include ::Gitlab::Utils::StrongMemoize @@ -31,7 +30,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES + @max_includes = MAX_INCLUDES yield self if block_given? end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index d3e7210b820..d82ca875e76 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -12,7 +12,7 @@ module Gitlab self.new(build, ttl: build.metadata_timeout).encoded end - def initialize(build, ttl: nil) + def initialize(build, ttl:) @build = build @ttl = ttl end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 4e01688a955..cfefa79d9e0 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -3,13 +3,27 @@ module Gitlab module Ci class JwtV2 < Jwt + DEFAULT_AUD = Settings.gitlab.base_url + + def self.for_build(build, aud: DEFAULT_AUD) + new(build, ttl: build.metadata_timeout, aud: aud).encoded + end + + def initialize(build, ttl:, aud:) + super(build, ttl: ttl) + + @aud = aud + end + private + attr_reader :aud + def reserved_claims super.merge( iss: Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: Settings.gitlab.base_url + aud: aud ) end end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index 00ca723b258..c76a4309779 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -23,8 +23,7 @@ module Gitlab ::Gitlab::Ci::Reports::Sbom::Source.new( type: :dependency_scanning, - data: data, - fingerprint: fingerprint + data: data ) end @@ -37,10 +36,6 @@ module Gitlab data.dig(*keys).present? end end - - def fingerprint - Digest::SHA256.hexdigest(data.to_json) - end end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index da7faaab6ff..0c117d5f214 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -44,31 +44,15 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - # We want validation to happen regardless of VALIDATE_SCHEMA - # CI variable. - # - # Previously it controlled BOTH validation and enforcement of - # schema validation result. - # - # After 15.0 we will enforce schema validation by default - # See: https://gitlab.com/groups/gitlab-org/-/epics/6968 - schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } - - if validate - schema_validation_passed = schema_validator.valid? + return true unless validate - # Validation warnings are errors - schema_validator.errors.each { |error| report.add_error('Schema', error) } - schema_validator.warnings.each { |warning| report.add_error('Schema', warning) } + schema_validation_passed = schema_validator.valid? - schema_validation_passed - else - # Validation warnings are warnings - schema_validator.errors.each { |error| report.add_warning('Schema', error) } - schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } + schema_validator.errors.each { |error| report.add_error('Schema', error) } + schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } + schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } - true - end + schema_validation_passed end def schema_validator @@ -216,7 +200,22 @@ module Gitlab external_id: scanner_data['id'], name: scanner_data['name'], vendor: scanner_data.dig('vendor', 'name'), - version: scanner_data.dig('version'))) + version: scanner_data.dig('version'), + primary_identifiers: create_scan_primary_identifiers)) + end + + # TODO: primary_identifiers should be initialized on the + # scan itself but we do not currently parse scans through `MergeReportsService` + def create_scan_primary_identifiers + return unless scan_data.is_a?(Hash) && scan_data.dig('primary_identifiers') + + scan_data.dig('primary_identifiers').map do |identifier| + ::Gitlab::Ci::Reports::Security::Identifier.new( + external_type: identifier['type'], + external_id: identifier['value'], + name: identifier['name'], + url: identifier['url']) + end end def create_identifiers(identifiers) diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb deleted file mode 100644 index 24613a441be..00000000000 --- a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Parsers - module Security - module Concerns - module DeprecatedSyntax - extend ActiveSupport::Concern - - included do - extend ::Gitlab::Utils::Override - - override :parse_report - end - - def report_data - @report_data ||= begin - data = super - - if data.is_a?(Array) - data = { - "version" => self.class::DEPRECATED_REPORT_VERSION, - "vulnerabilities" => data - } - end - - data - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb index e3c62614cd8..3d999f20f1e 100644 --- a/lib/gitlab/ci/parsers/security/sast.rb +++ b/lib/gitlab/ci/parsers/security/sast.rb @@ -5,10 +5,6 @@ module Gitlab module Parsers module Security class Sast < Common - include Security::Concerns::DeprecatedSyntax - - DEPRECATED_REPORT_VERSION = "1.2" - private def create_location(location_data) diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb index c6d95c1d391..175731b6b64 100644 --- a/lib/gitlab/ci/parsers/security/secret_detection.rb +++ b/lib/gitlab/ci/parsers/security/secret_detection.rb @@ -5,10 +5,6 @@ module Gitlab module Parsers module Security class SecretDetection < Common - include Security::Concerns::DeprecatedSyntax - - DEPRECATED_REPORT_VERSION = "1.2" - private def create_location(location_data) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 28d6620e5c4..627a1f58715 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..0fcab3cd8bb --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/cluster-image-scanning-report-format.json @@ -0,0 +1,980 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json new file mode 100644 index 00000000000..c08d0b45ced --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/container-scanning-report-format.json @@ -0,0 +1,914 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..e1ee91de23c --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/coverage-fuzzing-report-format.json @@ -0,0 +1,870 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json new file mode 100644 index 00000000000..ba2b31cf6aa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dast-report-format.json @@ -0,0 +1,1275 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json new file mode 100644 index 00000000000..2bf207ce2c2 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/dependency-scanning-report-format.json @@ -0,0 +1,978 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json new file mode 100644 index 00000000000..c3f3bf8265f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/sast-report-format.json @@ -0,0 +1,865 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json new file mode 100644 index 00000000000..9f7c4a45466 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.1/secret-detection-report-format.json @@ -0,0 +1,888 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.1" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..b753a44a3d4 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/cluster-image-scanning-report-format.json @@ -0,0 +1,980 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json new file mode 100644 index 00000000000..85671d03a27 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/container-scanning-report-format.json @@ -0,0 +1,912 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..33568a246fa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/coverage-fuzzing-report-format.json @@ -0,0 +1,870 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json new file mode 100644 index 00000000000..eb141f044d5 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dast-report-format.json @@ -0,0 +1,1275 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json new file mode 100644 index 00000000000..31905180019 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/dependency-scanning-report-format.json @@ -0,0 +1,978 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json new file mode 100644 index 00000000000..efc9715aafb --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/sast-report-format.json @@ -0,0 +1,865 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json new file mode 100644 index 00000000000..adbd01760d7 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.2/secret-detection-report-format.json @@ -0,0 +1,888 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.2" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 9c12d46cede..07a3aff1862 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,9 +11,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! + ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) + return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -27,13 +29,19 @@ module Gitlab private - def auto_cancelable_pipelines - project.all_pipelines.created_after(1.week.ago) + def auto_cancelable_pipelines(ff_enabled) + relation = project.all_pipelines + .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) - .id_not_in(pipeline.same_family_pipeline_ids) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled + + if ff_enabled + relation.id_not_in(pipeline.id) + else + relation.id_not_in(pipeline.same_family_pipeline_ids) + end end def auto_cancel_interruptible_pipelines(pipeline_ids) @@ -41,6 +49,14 @@ module Gitlab .id_in(pipeline_ids) .with_only_interruptible_builds .each do |cancelable_pipeline| + Gitlab::AppLogger.info( + class: self.class.name, + message: "Pipeline #{pipeline.id} auto-canceling pipeline #{cancelable_pipeline.id}", + canceled_pipeline_id: cancelable_pipeline.id, + canceled_by_pipeline_id: pipeline.id, + canceled_by_pipeline_source: pipeline.source + ) + # cascade_to_children not needed because we iterate through descendants here cancelable_pipeline.cancel_running( auto_canceled_by_pipeline_id: pipeline.id, diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 14c320f77bf..76d4a05bf30 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -121,11 +121,7 @@ module Gitlab end def observe_jobs_count_in_alive_pipelines - jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project) - project.all_pipelines.jobs_count_in_alive_pipelines - else - project.all_pipelines.builds_count_in_alive_pipelines - end + jobs_count = project.all_pipelines.jobs_count_in_alive_pipelines metrics.active_jobs_histogram .observe({ plan: project.actual_plan_name }, jobs_count) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a14dec48619..d41213ef6dd 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -9,15 +9,6 @@ module Gitlab include Chain::Helpers include ::Gitlab::Utils::StrongMemoize - SOURCES = [ - Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, - Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, - Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, - Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, - Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, - Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops - ].freeze - def perform! if pipeline_config&.exists? @pipeline.build_pipeline_config(content: pipeline_config.content) @@ -36,8 +27,6 @@ module Gitlab def pipeline_config strong_memoize(:pipeline_config) do - next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project) - ::Gitlab::Ci::ProjectConfig.new( project: project, sha: @pipeline.sha, custom_content: @command.content, @@ -45,24 +34,9 @@ module Gitlab ) end end - - def legacy_find_config - sources.each do |source| - config = source.new(@pipeline, @command) - return config if config.exists? - end - - nil - end - - def sources - SOURCES - end end end end end end end - -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb deleted file mode 100644 index 4947e2eb879..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class AutoDevops < Source - def content - strong_memoize(:content) do - next unless project&.auto_devops_enabled? - - template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) - YAML.dump('include' => [{ 'template' => template.full_name }]) - end - end - - def source - :auto_devops_source - end - - private - - def template_name - 'Auto-DevOps' - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb b/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb deleted file mode 100644 index 39ffa2d4e25..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/bridge.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Bridge < Source - def content - return unless @command.bridge - - @command.bridge.yaml_for_downstream - end - - def source - :bridge_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb deleted file mode 100644 index 092e7d43371..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class ExternalProject < Source - def content - strong_memoize(:content) do - next unless external_project_path? - - path_file, path_project, ref = extract_location_tokens - - config_location = { 'project' => path_project, 'file' => path_file } - config_location['ref'] = ref if ref.present? - - YAML.dump('include' => [config_location]) - end - end - - def source - :external_project_source - end - - private - - # Example: path/to/.gitlab-ci.yml@another-group/another-project - def external_project_path? - ci_config_path =~ /\A.+(yml|yaml)@.+\z/ - end - - # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname - def extract_location_tokens - path_file, path_project = ci_config_path.split('@', 2) - - if path_project.include? ":" - project, ref = path_project.split(':', 2) - [path_file, project, ref] - else - [path_file, path_project] - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb deleted file mode 100644 index 9954aedc4b7..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Parameter < Source - UnsupportedSourceError = Class.new(StandardError) - - def content - strong_memoize(:content) do - next unless command.content.present? - - command.content - end - end - - def source - :parameter_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb deleted file mode 100644 index 4990a5a6eb5..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Remote < Source - def content - strong_memoize(:content) do - next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) - - YAML.dump('include' => [{ 'remote' => ci_config_path }]) - end - end - - def source - :remote_source - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb deleted file mode 100644 index 0752b099d3d..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - class Repository < Source - def content - strong_memoize(:content) do - next unless file_in_repository? - - YAML.dump('include' => [{ 'local' => ci_config_path }]) - end - end - - def source - :repository_source - end - - private - - def file_in_repository? - return unless project - return unless @pipeline.sha - - project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present? - rescue GRPC::NotFound, GRPC::Internal - nil - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb deleted file mode 100644 index 69dca1568b6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Config - class Content - # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed. - class Source - include Gitlab::Utils::StrongMemoize - - DEFAULT_YAML_FILE = '.gitlab-ci.yml' - - attr_reader :command - - def initialize(pipeline, command) - @pipeline = pipeline - @command = command - end - - def exists? - strong_memoize(:exists) do - content.present? - end - end - - def content - raise NotImplementedError - end - - def source - raise NotImplementedError - end - - def project - @project ||= @pipeline.project - end - - def ci_config_path - @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb new file mode 100644 index 00000000000..8b26416edf7 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class ActiveJobs < Chain::Base + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Ci::Pipeline::Chain::Helpers + + LIMIT_NAME = :ci_active_jobs + MESSAGE = "Project exceeded the allowed number of jobs in active pipelines. Retry later." + + def perform! + return unless limits.exceeded?(LIMIT_NAME, count_jobs_in_alive_pipelines) + + error(MESSAGE, drop_reason: :job_activity_limit_exceeded) + + Gitlab::AppLogger.info( + class: self.class.name, + message: MESSAGE, + project_id: project.id, + plan: project.actual_plan_name) + end + + def break? + pipeline.errors.any? + end + + private + + def namespace + strong_memoize(:namespace) do + project.namespace + end + end + + def limits + strong_memoize(:limits) do + namespace.actual_limits + end + end + + def count_jobs_in_alive_pipelines + strong_memoize(:count_jobs_in_alive_pipelines) do + count_persisted_jobs_in_all_alive_pipelines + count_current_pipeline_jobs + end + end + + def count_current_pipeline_jobs + command.pipeline_seed.size + end + + def count_persisted_jobs_in_all_alive_pipelines + project.all_pipelines.jobs_count_in_alive_pipelines + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb deleted file mode 100644 index 3706dd0b9f6..00000000000 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Limit - class JobActivity < Chain::Base - def perform! - # to be overridden in EE - end - - def break? - false # to be overridden in EE - end - end - end - end - end - end -end - -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..4bec8355732 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -25,6 +25,8 @@ module Gitlab return error('Failed to build the pipeline!') end + set_pipeline_name + raise Populate::PopulateError if pipeline.persisted? end @@ -34,6 +36,15 @@ module Gitlab private + def set_pipeline_name + return if Feature.disabled?(:pipeline_name, pipeline.project) || + @command.yaml_processor_result.workflow_name.blank? + + name = @command.yaml_processor_result.workflow_name + + pipeline.build_pipeline_metadata(project: pipeline.project, title: name) + end + def stage_names # We filter out `.pre/.post` stages, as they alone are not considered # a complete pipeline: diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index de24bbf688b..e8a991026b5 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -91,7 +91,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) status = %w[success failed running canceled] - builds = pipeline.builds.latest + builds = pipeline.processables.latest .where(status: status).where.not(started_at: nil).order(:started_at) from_builds(builds) diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 44d905faced..4b7cbae5004 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -86,6 +86,7 @@ module Gitlab 'count' => values.size, 'min' => values.min, 'max' => values.max, + 'sum' => values.sum, 'avg' => values.sum / values.size } end.compact diff --git a/lib/gitlab/ci/processable_object_hierarchy.rb b/lib/gitlab/ci/processable_object_hierarchy.rb index 1122361e27e..c1531c3f4ab 100644 --- a/lib/gitlab/ci/processable_object_hierarchy.rb +++ b/lib/gitlab/ci/processable_object_hierarchy.rb @@ -20,12 +20,16 @@ module Gitlab def ancestor_conditions(cte) middle_table[:name].eq(objects_table[:name]).and( middle_table[:build_id].eq(cte.table[:id]) + ).and( + objects_table[:commit_id].eq(cte.table[:commit_id]) ) end def descendant_conditions(cte) middle_table[:build_id].eq(objects_table[:id]).and( middle_table[:name].eq(cte.table[:name]) + ).and( + objects_table[:commit_id].eq(cte.table[:commit_id]) ) end end diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index ea0fb8d4fbb..fbb8644c1b0 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -5,12 +5,11 @@ module Gitlab module Reports module Sbom class Source - attr_reader :source_type, :data, :fingerprint + attr_reader :source_type, :data - def initialize(type:, data:, fingerprint:) + def initialize(type:, data:) @source_type = type @data = data - @fingerprint = fingerprint end end end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 70f2919d38d..54b21da5436 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -69,6 +69,10 @@ module Gitlab replace_with!(::Security::MergeReportsService.new(self, other).execute) end + def primary_identifiers + scanners.values.flat_map(&:primary_identifiers).compact + end + def primary_scanner scanners.first&.second end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index 918df163ede..080ed3f834a 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -16,15 +16,16 @@ module Gitlab "semgrep" => 2 }.freeze - attr_accessor :external_id, :name, :vendor, :version + attr_accessor :external_id, :name, :vendor, :version, :primary_identifiers alias_method :key, :external_id - def initialize(external_id:, name:, vendor:, version:) + def initialize(external_id:, name:, vendor:, version:, primary_identifiers: nil) @external_id = external_id @name = name @vendor = vendor @version = version + @primary_identifiers = primary_identifiers end def to_hash diff --git a/lib/gitlab/ci/secure_files/cer.rb b/lib/gitlab/ci/secure_files/cer.rb new file mode 100644 index 00000000000..45d2898c29b --- /dev/null +++ b/lib/gitlab/ci/secure_files/cer.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class Cer + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata) + @filedata = filedata + @error = nil + end + + def certificate_data + OpenSSL::X509::Certificate.new(@filedata) + rescue OpenSSL::X509::CertificateError => err + @error = err.to_s + nil + end + strong_memoize_attr :certificate_data + + def metadata + return {} unless certificate_data + + { + issuer: issuer, + subject: subject, + id: id, + expires_at: expires_at + } + end + strong_memoize_attr :metadata + + private + + def expires_at + certificate_data.not_before + end + + def id + certificate_data.serial.to_s + end + + def issuer + X509Name.parse(certificate_data.issuer) + end + + def subject + X509Name.parse(certificate_data.subject) + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/mobile_provision.rb b/lib/gitlab/ci/secure_files/mobile_provision.rb new file mode 100644 index 00000000000..4ea74e20310 --- /dev/null +++ b/lib/gitlab/ci/secure_files/mobile_provision.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true +require 'cfpropertylist' + +module Gitlab + module Ci + module SecureFiles + class MobileProvision + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata) + @filedata = filedata + end + + def decoded_plist + p7 = OpenSSL::PKCS7.new(@filedata) + p7.verify(nil, OpenSSL::X509::Store.new, nil, OpenSSL::PKCS7::NOVERIFY) + p7.data + rescue ArgumentError, OpenSSL::PKCS7::PKCS7Error => err + @error = err.to_s + nil + end + strong_memoize_attr :decoded_plist + + def properties + list = CFPropertyList::List.new(data: decoded_plist, format: CFPropertyList::List::FORMAT_XML).value + CFPropertyList.native_types(list) + rescue CFFormatError, CFPlistError, CFTypeError => err + @error = err.to_s + nil + end + strong_memoize_attr :properties + + def metadata + return {} unless properties + + { + id: id, + expires_at: expires_at, + platforms: properties["Platform"], + team_name: properties['TeamName'], + team_id: properties['TeamIdentifier'], + app_name: properties['AppIDName'], + app_id: properties['Name'], + app_id_prefix: properties['ApplicationIdentifierPrefix'], + xcode_managed: properties['IsXcodeManaged'], + entitlements: properties['Entitlements'], + devices: properties['ProvisionedDevices'], + certificate_ids: certificate_ids + } + end + strong_memoize_attr :metadata + + private + + def id + properties['UUID'] + end + + def expires_at + properties['ExpirationDate'] + end + + def certificate_ids + return [] if developer_certificates.empty? + + developer_certificates.map { |c| c.metadata[:id] } + end + + def developer_certificates + certificates = properties['DeveloperCertificates'] + return if certificates.empty? + + certs = [] + certificates.each_with_object([]) do |cert, obj| + certs << Cer.new(cert) + end + + certs + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/p12.rb b/lib/gitlab/ci/secure_files/p12.rb new file mode 100644 index 00000000000..1006a4d05b2 --- /dev/null +++ b/lib/gitlab/ci/secure_files/p12.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class P12 + include Gitlab::Utils::StrongMemoize + + attr_reader :error + + def initialize(filedata, password = nil) + @filedata = filedata + @password = password + end + + def certificate_data + OpenSSL::PKCS12.new(@filedata, @password).certificate + rescue OpenSSL::PKCS12::PKCS12Error => err + @error = err.to_s + nil + end + strong_memoize_attr :certificate_data + + def metadata + return {} unless certificate_data + + { + issuer: issuer, + subject: subject, + id: serial, + expires_at: expires_at + } + end + strong_memoize_attr :metadata + + private + + def expires_at + certificate_data.not_before + end + + def serial + certificate_data.serial.to_s + end + + def issuer + X509Name.parse(certificate_data.issuer) + end + + def subject + X509Name.parse(certificate_data.subject) + end + end + end + end +end diff --git a/lib/gitlab/ci/secure_files/x509_name.rb b/lib/gitlab/ci/secure_files/x509_name.rb new file mode 100644 index 00000000000..659959b8ae5 --- /dev/null +++ b/lib/gitlab/ci/secure_files/x509_name.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class X509Name + def self.parse(x509_name) + x509_name.to_utf8.split(',').to_h { |a| a.split('=') } + rescue StandardError + {} + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index 3096af1b173..fbdaeecca5d 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -33,3 +33,8 @@ test: stage: test script: - ./runmytests.sh + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 4fe37ceaeaa..3379ce2f649 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -28,3 +28,8 @@ test: # If you need to run any migrations or configure the database, this # would be the point to do it. - lein test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 68b55b782cd..9584ec5deef 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -42,3 +42,8 @@ spec: minitest: script: - crystal test/spec_test.cr # change to the file(s) you execute for tests + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index acc4a9d2917..21dda92257e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -74,3 +74,8 @@ django-tests: - echo "GRANT ALL on *.* to '${MYSQL_USER}';"| mysql -u root --password="${MYSQL_ROOT_PASSWORD}" -h mysql # use python3 explicitly. see https://wiki.ubuntu.com/Python/3 - python3 manage.py test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 83ddce936e6..8b20c4cbccc 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -24,3 +24,8 @@ before_script: mix: script: - mix test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 021662ab416..7f81755348c 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -35,3 +35,8 @@ test: - $CI_PROJECT_DIR/coverage reports: junit: report.xml + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 603aede4d46..8cfea3e236f 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -28,3 +28,8 @@ compile: artifacts: paths: - mybinaries + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 08dc10d34b7..671925c5df6 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -1,3 +1,5 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: @@ -39,3 +41,8 @@ test: paths: - build - .gradle + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 03c8941169f..01697f67b89 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -46,3 +46,8 @@ before_script: build: script: - ./gradlew build + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index ce227bad19a..071eccbab0d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index ce227bad19a..071eccbab0d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index e278539d214..23efed212f8 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.85.29" + CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:0.87.0" needs: [] script: - export SOURCE_CODE=$PWD @@ -26,6 +26,11 @@ code_quality: echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " done } + - | + if [ -n "$CODECLIMATE_REGISTRY_USERNAME" ] && [ -n "$CODECLIMATE_REGISTRY_PASSWORD" ] && [ -n "$CODECLIMATE_PREFIX" ]; then + CODECLIMATE_REGISTRY=${CODECLIMATE_PREFIX%%/*} + docker login "$CODECLIMATE_REGISTRY" --username "$CODECLIMATE_REGISTRY_USERNAME" --password "$CODECLIMATE_REGISTRY_PASSWORD" + fi - docker pull --quiet "$CODE_QUALITY_IMAGE" - | docker run --rm \ @@ -38,6 +43,8 @@ code_quality: REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ CODECLIMATE_PREFIX \ + CODECLIMATE_REGISTRY_USERNAME \ + CODECLIMATE_REGISTRY_PASSWORD \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 539e1a6385d..d994ed70ea9 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 7cbc8e40b47..222f534387a 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 70f85382967..67057e916a8 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -50,6 +50,8 @@ dependency_scanning: artifacts: paths: - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 78fe108e8b9..7ad71625436 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index bc2e1fed0d4..10c843f60a6 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 34084272b29..d39b329c4b6 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -81,3 +81,8 @@ pages: # description page. # # [3]: https://hub.docker.com/_/julia/ + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 3a490012f3d..7bb2ea328e7 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -78,3 +78,8 @@ test: # set it in your package.json script # comment this out if you don't have a frontend test - npm test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index dfa46d7af61..347b811bc5d 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -14,26 +14,44 @@ # * Deploy built artifacts from master branch only. variables: - # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. - MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" - # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used + MAVEN_OPTS: >- + -Dhttps.protocols=TLSv1.2 + -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository + -Dorg.slf4j.simpleLogger.showDateTime=true + -Djava.awt.headless=true + + # As of Maven 3.3.0 instead of this you MAY define these options in `.mvn/maven.config` so the same config is used # when running from the command line. + # As of Maven 3.6.1, the use of `--no-tranfer-progress` (or `-ntp`) suppresses download and upload messages. The use + # of the `Slf4jMavenTransferListener` is no longer necessary. # `installAtEnd` and `deployAtEnd` are only effective with recent version of the corresponding plugins. - MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + MAVEN_CLI_OPTS: >- + --batch-mode + --errors + --fail-at-end + --show-version + --no-transfer-progress + -DinstallAtEnd=true + -DdeployAtEnd=true -# This template uses jdk8 for verifying and deploying images -image: maven:3.3.9-jdk-8 +# This template uses the latest Maven 3 release, e.g., 3.8.6, and OpenJDK 8 (LTS) +# for verifying and deploying images +# Maven 3.8.x REQUIRES HTTPS repositories. +# See https://maven.apache.org/docs/3.8.1/release-notes.html#how-to-fix-when-i-get-a-http-repository-blocked for more. +image: maven:3-openjdk-8 # Cache downloaded dependencies and plugins between builds. # To keep cache across branches add 'key: "$CI_JOB_NAME"' +# Be aware that `mvn deploy` will install the built jar into this repository. If you notice your cache size +# increasing, consider adding `-Dmaven.install.skip=true` to `MAVEN_OPTS` or in `.mvn/maven.config` cache: paths: - .m2/repository # For merge requests do not `deploy` but only run `verify`. # See https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html -.verify: &verify +.verify: stage: test script: - 'mvn $MAVEN_CLI_OPTS verify' @@ -43,19 +61,20 @@ cache: # Verify merge requests using JDK8 verify:jdk8: - <<: *verify + extends: + - .verify -# To deploy packages from CI, create a ci_settings.xml file +# To deploy packages from CI, create a `ci_settings.xml` file # For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for more details. # Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. -# For `master` branch run `mvn deploy` automatically. +# For `master` or `main` branch run `mvn deploy` automatically. deploy:jdk8: stage: deploy script: - - if [ ! -f ci_settings.xml ]; - then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; + - if [ ! -f ci_settings.xml ]; then + echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; fi - - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' + - 'mvn $MAVEN_CLI_OPTS deploy --settings ci_settings.xml' only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 65db649e22f..e34bb8307f4 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -49,3 +49,8 @@ debug: - msbuild /p:Configuration="Debug" /p:Platform="Any CPU" /p:OutputPath="./../../build/debug/" "MyProject.sln" - mono packages/NUnit.ConsoleRunner.3.6.0/tools/nunit3-console.exe build/debug/MyProject.Test.dll + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 7a4f7ed628b..649d525df76 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -33,3 +33,8 @@ test_db: script: - npm install - node ./specs/start.js ./specs/db-postgres.spec.js + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 12640d28d29..0604438e0aa 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -42,3 +42,8 @@ variables: test: script: - vendor/bin/phpunit --configuration phpunit.xml --coverage-text --colors=never + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 55cf22b6601..51d2273d41d 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index 2f518d667a5..e577a489c55 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -11,6 +11,7 @@ pages: - apk update && apk add doxygen - doxygen doxygen/Doxyfile - mv doxygen/documentation/html/ public/ + environment: production artifacts: paths: - public diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 9da50439be8..88ed73b41e7 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -20,3 +20,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 85f90984045..9f3ba6d5dd4 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -6,6 +6,7 @@ # Full project: https://gitlab.com/pages/plain-html pages: stage: deploy + environment: production script: - mkdir .public - cp -r ./* .public diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 9e48ac9fcdc..aa86ad2a6ad 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index a6f94a4d80e..b1617e9239c 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -11,6 +11,7 @@ pages: - npm install hexo-cli -g - test -e package.json && npm install - hexo generate + environment: production artifacts: paths: - public diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 591eebf9cd6..d6f6e94526e 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -27,3 +27,4 @@ pages: only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 59e55efaee0..fba4afca9ed 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -21,6 +21,7 @@ test: pages: stage: deploy + environment: production script: - pip install hyde - hyde gen -d public diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 8e15570fd1a..57e3ced4dc2 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -29,6 +29,7 @@ before_script: # This build job produced the output directory of your site pages: + environment: production script: - jbake . public artifacts: diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index e0ad2e55f7d..8b07454af24 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -36,3 +36,4 @@ pages: only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 26fac92d0dc..ad083fcc5db 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -40,3 +40,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 9b5c1198c6c..e86337ae23c 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index d97f0b7beb7..a49e95b62c8 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -19,3 +19,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 17ce0ef3659..d8f036ab4ed 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -31,3 +31,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index a3ce96da244..b0511abd109 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -15,3 +15,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4abdf66a21c..c89050eede7 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -18,3 +18,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 7d52a407848..3721344b21e 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -13,3 +13,4 @@ pages: artifacts: paths: - public/ + environment: production diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 961941ac4d0..00efcfa1b32 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -32,3 +32,4 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 191d5b6b11c..febbb36d834 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -55,3 +55,8 @@ pages: - public rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index a83f84da818..b9823444db2 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -40,3 +40,8 @@ test:cargo: # when: always # reports: # junit: $CI_PROJECT_DIR/tests/*.xml + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index 26efe7a8908..ce5d5937896 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -30,3 +30,8 @@ test: script: # Execute your project's tests - sbt clean test + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8d6c191edc4..f12efa1db34 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -40,6 +40,19 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $CI_GITLAB_FIPS_MODE == "true" variables: @@ -55,5 +68,3 @@ apifuzzer_fuzz: - gl-*.log reports: api_fuzzing: gl-api-fuzzing-report.json - -# end diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index d4b6a252b25..d933007ec61 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -31,15 +31,15 @@ coverage_fuzzing_unlicensed: stage: fuzz allow_failure: true before_script: - - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN - - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN - - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH - - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID + - export COVFUZZ_JOB_TOKEN="$CI_JOB_TOKEN" + - export COVFUZZ_PRIVATE_TOKEN="$CI_PRIVATE_TOKEN" + - export COVFUZZ_PROJECT_PATH="$CI_PROJECT_PATH" + - export COVFUZZ_PROJECT_ID="$CI_PROJECT_ID" - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 - chmod a+x gitlab-cov-fuzz - export REGRESSION=true - - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi; + - if [[ "$CI_COMMIT_BRANCH" = "$COVFUZZ_BRANCH" ]]; then REGRESSION=false; fi; artifacts: paths: - corpus diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..feed4c47157 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,64 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml + +# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables + +variables: + # Which branch we want to run full fledged long running fuzzing jobs. + # All others will run fuzzing regression + COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH" + # This is using semantic version and will always download latest v3 gitlab-cov-fuzz release + COVFUZZ_VERSION: v3 + # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries + # to their own servers + COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" + + +coverage_fuzzing_unlicensed: + stage: .pre + allow_failure: true + rules: + - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null + script: + - echo "ERROR Your GitLab project is missing licensing for Coverage Fuzzing" && exit 1 + +.fuzz_base: + stage: fuzz + allow_failure: true + before_script: + - export COVFUZZ_JOB_TOKEN="$CI_JOB_TOKEN" + - export COVFUZZ_PRIVATE_TOKEN="$CI_PRIVATE_TOKEN" + - export COVFUZZ_PROJECT_PATH="$CI_PROJECT_PATH" + - export COVFUZZ_PROJECT_ID="$CI_PROJECT_ID" + - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi + - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 + - chmod a+x gitlab-cov-fuzz + - export REGRESSION=true + - if [[ "$CI_COMMIT_BRANCH" = "$COVFUZZ_BRANCH" ]]; then REGRESSION=false; fi; + artifacts: + paths: + - corpus + - crashes + - gl-coverage-fuzzing-report.json + reports: + coverage_fuzzing: gl-coverage-fuzzing-report.json + when: always + rules: + - if: $COVFUZZ_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 8aabf20c5df..a28914d082f 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -40,6 +40,19 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $CI_GITLAB_FIPS_MODE == "true" variables: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 9d3b1f4316e..50e9bb5431d 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -52,6 +52,19 @@ dast: - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && $REVIEW_DISABLED when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdast\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index 3c4533d603e..2a5ac539a42 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -38,3 +38,8 @@ archive_project: - ios_11-3 - xcode_9-3 - macos_10-13 + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index b8d284532bd..5fcbb251672 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -111,3 +111,8 @@ tests: # (e.g. integration tests, unit tests etc). script: - 'dotnet test --no-restore' + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 58e840da713..400226b415d 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -36,3 +36,8 @@ build: artifacts: paths: - ./*.ipa + +deploy: + stage: deploy + script: echo "Define your deployment script!" + environment: production diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c5664ef1cfb..2dc7bbc391e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -140,7 +140,7 @@ module Gitlab def being_watched? Gitlab::Redis::SharedState.with do |redis| - redis.exists(being_watched_cache_key) + redis.exists?(being_watched_cache_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 528d72c9bcc..cf5f04215ad 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -11,6 +11,7 @@ module Gitlab @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) @group_variables_builder = Builder::Group.new(project&.group) + @release_variables_builder = Builder::Release.new(release) end def scoped_variables(job, environment:, dependencies:) @@ -28,6 +29,7 @@ module Gitlab variables.concat(secret_project_variables(environment: environment)) variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) + variables.concat(release_variables) end end @@ -106,18 +108,26 @@ module Gitlab end end + def release_variables + strong_memoize(:release_variables) do + release_variables_builder.variables + end + end + private attr_reader :pipeline attr_reader :instance_variables_builder attr_reader :project_variables_builder attr_reader :group_variables_builder + attr_reader :release_variables_builder delegate :project, to: :pipeline def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_JOB_NAME', value: job.name) + variables.append(key: 'CI_JOB_NAME_SLUG', value: job_name_slug(job)) variables.append(key: 'CI_JOB_STAGE', value: job.stage_name) variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action? variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request @@ -145,6 +155,10 @@ module Gitlab end end + def job_name_slug(job) + job.name && Gitlab::Utils.slugify(job.name) + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) @@ -166,6 +180,12 @@ module Gitlab container[args] = yield end end + + def release + return unless @pipeline.tag? + + project.releases.find_by_tag(@pipeline.ref) + end end end end diff --git a/lib/gitlab/ci/variables/builder/release.rb b/lib/gitlab/ci/variables/builder/release.rb new file mode 100644 index 00000000000..25f3d8446c5 --- /dev/null +++ b/lib/gitlab/ci/variables/builder/release.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Release + include Gitlab::Utils::StrongMemoize + + attr_reader :release + + DESCRIPTION_LIMIT = 1024 + + def initialize(release) + @release = release + end + + def variables + strong_memoize(:variables) do + ::Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless release + + if release.description + variables.append( + key: 'CI_RELEASE_DESCRIPTION', + value: release.description.truncate(DESCRIPTION_LIMIT), + raw: true) + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 52673d03e69..b6d6e1a3e5f 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,7 +72,7 @@ module Gitlab Collection.new(@variables.reject(&block)) end - def expand_value(value, keep_undefined: false, expand_file_vars: true) + def expand_value(value, keep_undefined: false, expand_file_vars: true, project: nil) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -88,6 +88,16 @@ module Gitlab if variable # VARIABLE_NAME is an existing variable next variable.value unless variable.file? + # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 + if project + # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` + # when the variables are sent to Runner. + Gitlab::AppJsonLogger.info( + event: 'file_variable_is_referenced_in_another_variable', + project_id: project.id + ) + end + expand_file_vars ? variable.value : full_match elsif keep_undefined full_match # we do not touch the variable definition @@ -97,7 +107,7 @@ module Gitlab end end - def sort_and_expand_all(keep_undefined: false, expand_file_vars: true) + def sort_and_expand_all(keep_undefined: false, expand_file_vars: true, project: nil) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? @@ -112,7 +122,8 @@ module Gitlab # expand variables as they are added variable = item.to_runner_variable variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_vars: expand_file_vars) + expand_file_vars: expand_file_vars, + project: project) new_collection.append(variable) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f203f88442d..5c3864362da 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -32,18 +32,16 @@ module Gitlab end end - def stage_builds_attributes(stage) - jobs.values - .select { |job| job[:stage] == stage } - .map { |job| build_attributes(job[:name]) } - end - def workflow_rules @workflow_rules ||= @ci_config.workflow_rules end + def workflow_name + @workflow_name ||= @ci_config.workflow_name&.strip + end + def root_variables - @root_variables ||= transform_to_array(variables) + @root_variables ||= transform_to_array(@ci_config.variables) end def jobs @@ -58,6 +56,38 @@ module Gitlab @included_templates ||= @ci_config.included_templates end + def variables_with_data + @ci_config.variables_with_data + end + + def yaml_variables_for(job_name) + job = jobs[job_name] + + return [] unless job + + Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: root_variables, + to: job[:job_variables], + inheritance: job.fetch(:root_variables_inheritance, true) + ) + end + + def stage_for(job_name) + jobs.dig(job_name, :stage) + end + + def config_metadata + @ci_config&.metadata || {} + end + + private + + def stage_builds_attributes(stage) + jobs.values + .select { |job| job[:stage] == stage } + .map { |job| build_attributes(job[:name]) } + end + def build_attributes(name) job = jobs.fetch(name.to_sym, {}) @@ -103,36 +133,6 @@ module Gitlab }.compact }.compact end - def variables_with_data - @ci_config.variables_with_data - end - - def yaml_variables_for(job_name) - job = jobs[job_name] - - return [] unless job - - Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( - from: root_variables, - to: job[:job_variables], - inheritance: job.fetch(:root_variables_inheritance, true) - ) - end - - def stage_for(job_name) - jobs.dig(job_name, :stage) - end - - def config_metadata - @ci_config&.metadata || {} - end - - private - - def variables - @variables ||= @ci_config.variables - end - def release(job) job[:release] end diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index be7d26fed4e..415f6f77214 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -50,12 +50,6 @@ module Gitlab variables.values.flatten(1).all?(&method(:validate_alphanumeric)) end - def validate_string_or_hash_value_variables(variables, allowed_value_data) - variables.is_a?(Hash) && - variables.keys.all?(&method(:validate_alphanumeric)) && - variables.values.all? { |value| validate_string_or_hash_value_variable(value, allowed_value_data) } - end - def validate_alphanumeric(value) validate_string(value) || validate_integer(value) end @@ -68,14 +62,6 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end - def validate_string_or_hash_value_variable(value, allowed_value_data) - if value.is_a?(Hash) - (value.keys - allowed_value_data).empty? && value.values.all?(&method(:validate_alphanumeric)) - else - validate_alphanumeric(value) - end - end - def validate_regexp(value) Gitlab::UntrustedRegexp::RubySyntax.valid?(value) end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 337cfbc5287..b88a6766d92 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -44,7 +44,7 @@ module Gitlab mutually_exclusive_keys = value.try(:keys).to_a & options[:in] if mutually_exclusive_keys.length > 1 - record.errors.add(attribute, "please use only one the following keys: " + + record.errors.add(attribute, "please use only one of the following keys: " + mutually_exclusive_keys.join(', ')) end end @@ -304,15 +304,12 @@ module Gitlab end end - # This will be removed with the FF `ci_variables_refactoring_to_variable`. class VariablesValidator < ActiveModel::EachValidator include LegacyValidationHelpers def validate_each(record, attribute, value) if options[:array_values] validate_key_array_values(record, attribute, value) - elsif options[:allowed_value_data] - validate_key_hash_values(record, attribute, value, options[:allowed_value_data]) else validate_key_values(record, attribute, value) end @@ -329,12 +326,6 @@ module Gitlab record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array') end end - - def validate_key_hash_values(record, attribute, value, allowed_value_data) - unless validate_string_or_hash_value_variables(value, allowed_value_data) - record.errors.add(attribute, 'should be a hash of key value pairs, value can be a hash') - end - end end class AlphanumericValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index 54320b7ff9a..64950fb4eef 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -5,22 +5,29 @@ module Gitlab module ExternalDatabaseChecker extend self + PG_REQUIREMENTS_LINK = + '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' + def check - return [] if ApplicationRecord.database.postgresql_minimum_supported_version? + unsupported_database = Gitlab::Database + .database_base_models + .map { |_, model| Gitlab::Database::Reflection.new(model) } + .reject(&:postgresql_minimum_supported_version?) - [ + unsupported_database.map do |database| { type: 'warning', message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % { - pg_version_current: ApplicationRecord.database.version, - pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, - pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' - } + 'see %{pg_requirements_url} for details.') % \ + { + pg_version_current: database.version, + pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, + pg_requirements_url: PG_REQUIREMENTS_LINK + } } - ] + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 320ebe5e80f..a75c7c539ae 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -63,6 +63,7 @@ module Gitlab def hook_attrs(pipeline) { id: pipeline.id, + iid: pipeline.iid, ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 45f52765d0f..92cafd1d00e 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -67,11 +67,11 @@ module Gitlab end event :finish do - transition any => :finished + transition [:paused, :finished, :active, :finalizing] => :finished end event :failure do - transition any => :failed + transition [:failed, :finalizing, :active] => :failed end event :finalize do diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5725d7a4503..c4a9cf8b80f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -99,6 +99,7 @@ ci_pipeline_messages: :gitlab_ci ci_pipeline_schedules: :gitlab_ci ci_pipeline_schedule_variables: :gitlab_ci ci_pipelines_config: :gitlab_ci +ci_pipeline_metadata: :gitlab_ci ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci @@ -263,6 +264,8 @@ incident_management_oncall_shifts: :gitlab_main incident_management_pending_alert_escalations: :gitlab_main incident_management_pending_issue_escalations: :gitlab_main incident_management_timeline_events: :gitlab_main +incident_management_timeline_event_tags: :gitlab_main +incident_management_timeline_event_tag_links: :gitlab_main index_statuses: :gitlab_main in_product_marketing_emails: :gitlab_main insights: :gitlab_main @@ -389,6 +392,7 @@ packages_nuget_dependency_link_metadata: :gitlab_main packages_nuget_metadata: :gitlab_main packages_package_file_build_infos: :gitlab_main packages_package_files: :gitlab_main +packages_rpm_repository_files: :gitlab_main packages_packages: :gitlab_main packages_pypi_metadata: :gitlab_main packages_rubygems_metadata: :gitlab_main @@ -447,6 +451,7 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main +project_wiki_repository_states: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main prometheus_metrics: :gitlab_main @@ -545,6 +550,7 @@ user_group_callouts: :gitlab_main user_project_callouts: :gitlab_main user_highest_roles: :gitlab_main user_interacted_projects: :gitlab_main +user_phone_number_validations: :gitlab_main user_permission_export_uploads: :gitlab_main user_preferences: :gitlab_main users: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 40b76a1c028..0881025b425 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -105,16 +105,25 @@ module Gitlab def read_write connection = nil transaction_open = nil + attempts = 3 + + if prevent_load_balancer_retries_in_transaction? + attempts = 1 if pool.connection.transaction_open? + end + # In the event of a failover the primary may be briefly unavailable. # Instead of immediately grinding to a halt we'll retry the operation # a few times. - retry_with_backoff do + # It is not possible preserve transaction state during a retry, so we do not retry in that case. + retry_with_backoff(attempts: attempts) do |attempt| connection = pool.connection transaction_open = connection.transaction_open? yield connection rescue StandardError => e - if transaction_open && connection_error?(e) + # No leaking will happen on the final attempt. Leaks are caused by subsequent retries + not_final_attempt = attempt && attempt < attempts + if transaction_open && connection_error?(e) && not_final_attempt ::Gitlab::Database::LoadBalancing::Logger.warn( event: :transaction_leak, message: 'A write transaction has leaked during database fail-over' @@ -171,7 +180,7 @@ module Gitlab end # Yields a block, retrying it upon error using an exponential backoff. - def retry_with_backoff(retries = 3, time = 2) + def retry_with_backoff(attempts: 3, time: 2) # In CI we only use the primary, but databases may not always be # available (or take a few seconds to become available). Retrying in # this case can slow down CI jobs. In addition, retrying with _only_ @@ -183,12 +192,12 @@ module Gitlab # replicas were configured. return yield if primary_only? - retried = 0 + attempt = 1 last_error = nil - while retried < retries + while attempt <= attempts begin - return yield + return yield attempt # Yield the current attempt count rescue StandardError => error raise error unless connection_error?(error) @@ -198,7 +207,7 @@ module Gitlab last_error = error sleep(time) - retried += 1 + attempt += 1 time **= 2 end end @@ -332,6 +341,10 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end + + def prevent_load_balancer_retries_in_transaction? + Gitlab::Utils.to_boolean(ENV['PREVENT_LOAD_BALANCER_RETRIES_IN_TRANSACTION'], default: false) + end end end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 5d91292b8de..3180289ec69 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -97,8 +97,14 @@ module Gitlab end def databases_in_sync?(wal_locations) + locations = if Feature.enabled?(:indifferent_wal_location_keys) + wal_locations.with_indifferent_access + else + wal_locations + end + ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = wal_locations[lb.name]) + if (location = locations[lb.name]) lb.select_up_to_date_host(location) else # If there's no entry for a load balancer it means the Sidekiq diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index e574422ce11..df40e3b3868 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -296,12 +296,11 @@ module Gitlab with_lock_retries do execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order - execute <<-EOF.strip_heredoc ALTER TABLE #{source} ADD CONSTRAINT #{options[:name]} - FOREIGN KEY (#{options[:column]}) - REFERENCES #{target} (#{target_column}) + FOREIGN KEY (#{multiple_columns(options[:column])}) + REFERENCES #{target} (#{multiple_columns(target_column)}) #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -355,7 +354,7 @@ module Gitlab # - For standard rails foreign keys the prefix is `fk_rails_` # def concurrent_foreign_key_name(table, column, prefix: 'fk_') - identifier = "#{table}_#{column}_fk" + identifier = "#{table}_#{multiple_columns(column, separator: '_')}_fk" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) "#{prefix}#{hashed_identifier}" @@ -1503,6 +1502,26 @@ into similar problems in the future (e.g. when new tables are created). SQL end + def drop_constraint(table_name, constraint_name, cascade: false) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} + SQL + end + + def add_primary_key_using_index(table_name, pk_name, index_to_use) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)} + SQL + end + + def swap_primary_key(table_name, primary_key_name, index_to_use) + with_lock_retries(raise_on_exhaustion: true) do + drop_constraint(table_name, primary_key_name, cascade: true) + add_primary_key_using_index(table_name, primary_key_name, index_to_use) + end + end + alias_method :unswap_primary_key, :swap_primary_key + def drop_sequence(table_name, column_name, sequence_name) execute <<~SQL ALTER TABLE #{quote_table_name(table_name)} ALTER COLUMN #{quote_column_name(column_name)} DROP DEFAULT; @@ -1519,6 +1538,14 @@ into similar problems in the future (e.g. when new tables are created). private + def multiple_columns(columns, separator: ', ') + Array.wrap(columns).join(separator) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end + def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) unless table_exists?(table) raise "Table #{table} does not exist" diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index 76982a9da9b..dbb85bad95c 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -4,10 +4,11 @@ module Gitlab module Database module Migrations class BaseBackgroundRunner - attr_reader :result_dir + attr_reader :result_dir, :connection - def initialize(result_dir:) + def initialize(result_dir:, connection:) @result_dir = result_dir + @connection = connection end def jobs_by_migration_name @@ -45,7 +46,7 @@ module Gitlab instrumentation.observe(version: nil, name: batch_names.next, - connection: ActiveRecord::Migration.connection) do + connection: connection) do run_job(j) end end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 4404b5bf961..85dc6051c7c 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -6,40 +6,68 @@ module Gitlab class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' - SCHEMA_VERSION = 3 # Version of the output format produced by the runner + SCHEMA_VERSION = 4 # Version of the output format produced by the runner class << self - def up - Runner.new(direction: :up, migrations: migrations_for_up, result_dir: BASE_RESULT_DIR.join('up')) + def up(database:, legacy_mode: false) + within_context_for_database(database) do + Runner.new(direction: :up, database: database, migrations: migrations_for_up(database), legacy_mode: legacy_mode) + end end - def down - Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) + def down(database:, legacy_mode: false) + within_context_for_database(database) do + Runner.new(direction: :down, database: database, migrations: migrations_for_down(database), legacy_mode: legacy_mode) + end end def background_migrations TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) end - def batched_background_migrations(for_database:) + def batched_background_migrations(for_database:, legacy_mode: false) runner = nil + result_dir = if legacy_mode + BASE_RESULT_DIR.join('background_migrations') + else + BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') + end + # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: BASE_RESULT_DIR.join('background_migrations'), connection: connection) + .new(result_dir: result_dir, connection: connection) end runner end def migration_context - @migration_context ||= ApplicationRecord.connection.migration_context + # We're mirroring rails internal migration code, which requires that + # ActiveRecord::Base has connected to the current database. The correct database is chosen by + # within_context_for_database + ActiveRecord::Base.connection.migration_context # rubocop:disable Database/MultipleDatabases + end + + # rubocop:disable Database/MultipleDatabases + def within_context_for_database(database) + original_db_config = ActiveRecord::Base.connection_db_config + # The config only works if passed a string + db_config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: database.to_s) + raise ArgumentError, "Cannot find a database configuration for #{database}" unless db_config + + ActiveRecord::Base.establish_connection(db_config) # rubocop:disable Database/EstablishConnection + + yield + ensure + ActiveRecord::Base.establish_connection(original_db_config) # rubocop:disable Database/EstablishConnection end + # rubocop:enable Database/MultipleDatabases private - def migrations_for_up + def migrations_for_up(database) existing_versions = migration_context.get_all_versions.to_set migration_context.migrations.reject do |migration| @@ -51,7 +79,7 @@ module Gitlab `git diff --name-only origin/HEAD...HEAD db/post_migrate db/migrate`.split("\n") end - def migrations_for_down + def migrations_for_down(database) versions_this_branch = migration_file_names_this_branch.map do |m_name| m_name.match(%r{^db/(post_)?migrate/(\d+)}) { |m| m.captures[1]&.to_i } end.to_set @@ -65,14 +93,21 @@ module Gitlab attr_reader :direction, :result_dir, :migrations - delegate :migration_context, to: :class + delegate :migration_context, :within_context_for_database, to: :class - def initialize(direction:, migrations:, result_dir:) + def initialize(direction:, database:, migrations:, legacy_mode: false) raise "Direction must be up or down" unless %i[up down].include?(direction) @direction = direction @migrations = migrations - @result_dir = result_dir + @result_dir = if legacy_mode + BASE_RESULT_DIR.join(direction.to_s) + else + BASE_RESULT_DIR.join(database.to_s, direction.to_s) + end + + @database = database + @legacy_mode = legacy_mode end def run @@ -86,14 +121,22 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: result_dir) - sorted_migrations.each do |migration| - instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do - ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run + within_context_for_database(@database) do + sorted_migrations.each do |migration| + instrumentation.observe(version: migration.version, name: migration.name, connection: ActiveRecord::Migration.connection) do + ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run + end end end ensure metadata_filename = File.join(result_dir, METADATA_FILENAME) - File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) + version = if @legacy_mode + 3 + else + SCHEMA_VERSION + end + + File.write(metadata_filename, { database: @database.to_s, version: version }.to_json) # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 6da2e098d43..65db330b1a6 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -5,7 +5,7 @@ module Gitlab module Migrations class TestBackgroundRunner < BaseBackgroundRunner def initialize(result_dir:) - super(result_dir: result_dir) + super(result_dir: result_dir, connection: ActiveRecord::Migration.connection) @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index c27ae6a2c5d..46855ca1921 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -5,65 +5,73 @@ module Gitlab module Migrations class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - attr_reader :connection def initialize(result_dir:, connection:) - super(result_dir: result_dir) + super(result_dir: result_dir, connection: connection) @connection = connection end def jobs_by_migration_name - Gitlab::Database::BackgroundMigration::BatchedMigration - .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing - .to_h do |migration| - batching_strategy = migration.batch_class.new(connection: connection) - - smallest_batch_start = migration.next_min_value - - table_max_value = define_batchable_model(migration.table_name, connection: connection) - .maximum(migration.column_name) - - largest_batch_start = table_max_value - migration.batch_size - - # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 - # to pick actual batches to sample. - variance = largest_batch_start - smallest_batch_start - - batch_starts = uniform_fractions - .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start - .map { |frac| (variance * frac).to_i + smallest_batch_start } - - # Track previously run batches so that we stop sampling if a new batch would intersect an older one - completed_batches = [] - - jobs_to_sample = batch_starts - # Stop sampling if a batch would intersect a previous batch - .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } - .map do |batch_start| - next_bounds = batching_strategy.next_batch( - migration.table_name, - migration.column_name, - batch_min_value: batch_start, - batch_size: migration.batch_size, - job_arguments: migration.job_arguments - ) - - batch_min, batch_max = next_bounds - - job = migration.create_batched_job!(batch_min, batch_max) - - completed_batches << (batch_min..batch_max) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration + .executable + .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .to_h do |migration| + batching_strategy = migration.batch_class.new(connection: connection) + + smallest_batch_start = migration.next_min_value + + table_max_value = define_batchable_model(migration.table_name, connection: connection) + .maximum(migration.column_name) + + largest_batch_start = table_max_value - migration.batch_size + + # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 + # to pick actual batches to sample. + variance = largest_batch_start - smallest_batch_start + + batch_starts = uniform_fractions + .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start + .map { |frac| (variance * frac).to_i + smallest_batch_start } + + # Track previously run batches so that we stop sampling if a new batch would intersect an older one + completed_batches = [] + + jobs_to_sample = batch_starts + # Stop sampling if a batch would intersect a previous batch + .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } + .map do |batch_start| + # The current block is lazily evaluated as part of the jobs_to_sample enumerable + # so it executes after the enclosing using_connection block has already executed + # Therefore we need to re-associate with the explicit connection again + Gitlab::Database::SharedModel.using_connection(connection) do + next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: batch_start, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + ) + + batch_min, batch_max = next_bounds + + job = migration.create_batched_job!(batch_min, batch_max) + + completed_batches << (batch_min..batch_max) + + job + end + end - job + [migration.job_class_name, jobs_to_sample] end - - [migration.job_class_name, jobs_to_sample] end end def run_job(job) - Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + end end def uniform_fractions diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb index f45cf02ec9b..23a8dc0b44f 100644 --- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -6,8 +6,6 @@ module Gitlab class ConvertTableToFirstListPartition UnableToPartition = Class.new(StandardError) - include Gitlab::Database::MigrationHelpers - SQL_STATEMENT_SEPARATOR = ";\n\n" attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value @@ -175,9 +173,21 @@ module Gitlab def alter_sequence_statements(old_table:, new_table:) sequences_owned_by(old_table).map do |seq_info| seq_name, column_name = seq_info.values_at(:name, :column_name) - <<~SQL.chomp + + statement_parts = [] + + # If a different user owns the old table, the conversion process will fail to reassign the sequence + # ownership to the new parent table (as it will be owned by the current user). + # Force the old table to be owned by the current user in that case. + unless current_user_owns_table?(old_table) + statement_parts << set_current_user_owns_table_statement(old_table) + end + + statement_parts << <<~SQL.chomp ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} SQL + + statement_parts.join(SQL_STATEMENT_SEPARATOR) end end @@ -208,6 +218,23 @@ module Gitlab { name: name, column_name: column_name } end end + + def table_owner(table_name) + connection.select_value(<<~SQL, nil, [table_name]) + SELECT tableowner FROM pg_tables WHERE tablename = $1 + SQL + end + + def current_user_owns_table?(table_name) + current_user = connection.select_value('select current_user') + table_owner(table_name) == current_user + end + + def set_current_user_owns_table_statement(table_name) + <<~SQL.chomp + ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER + SQL + end end end end diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 5f321ece962..9a52a4f6e23 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -8,7 +8,7 @@ module Gitlab QueryAnalyzerError = Class.new(Exception) # rubocop:disable Lint/InheritException def self.suppressed? - Thread.current[self.suppress_key] + Thread.current[self.suppress_key] || @suppress_in_rspec end def self.requires_tracking?(parsed) @@ -19,6 +19,20 @@ module Gitlab Thread.current[self.suppress_key] = value end + # The other suppress= method stores the + # value in Thread.current because it is + # meant to work in a multi-threaded puma + # environment but this does not work + # correctly in capybara tests where we + # suppress in the rspec runner context but + # this does not take effect in the puma + # thread. As such we just suppress + # globally in RSpec since we don't run + # different tests concurrently. + class << self + attr_writer :suppress_in_rspec + end + def self.with_suppressed(value = true, &blk) previous = self.suppressed? self.suppress = value diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 33c965cb150..6c4e46728d4 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -124,7 +124,11 @@ module Gitlab # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-servers # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-servers#managing-your-server # this database is present on both Flexible and Single server, so we should check the former first. - 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" } + 'Azure Database for PostgreSQL - Single Server' => { statement: "SELECT datname FROM pg_database WHERE datname = 'azure_maintenance'" }, + # Based on + # - https://cloud.google.com/sql/docs/postgres/flags + # running a query to detect flag names that begin with 'alloydb + 'AlloyDB for PostgreSQL' => { statement: "SELECT name FROM pg_settings WHERE name LIKE 'alloydb%'" } }.each do |flavor, conditions| return flavor if connection.execute(conditions[:statement]).to_a.present? rescue ActiveRecord::StatementInvalid => e diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index f96de13006f..2c7ca28942e 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -71,12 +71,17 @@ module Gitlab "#{type}_#{hashed_identifier}" end - def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new( + def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) + merged_args = { connection: connection, klass: self.class, - logger: Gitlab::BackgroundMigration::Logger - ).run(&block) + logger: Gitlab::BackgroundMigration::Logger, + allow_savepoints: true + }.merge(kwargs) + + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) end def assert_not_in_transaction_block(scope:) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 084ce63e36a..d6f5e45c034 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -82,16 +82,6 @@ module Gitlab private - def expiration - return 1.day unless Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) - - if Feature.enabled?(:highlight_diffs_short_renewable_expiration, diffable.project) - EXPIRATION - else - 8.hours - end - end - def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) @@ -147,7 +137,7 @@ module Gitlab end # HSETs have to have their expiration date manually updated - pipeline.expire(key, expiration) + pipeline.expire(key, EXPIRATION) end record_memory_usage(fetch_memory_usage(redis, key)) @@ -197,14 +187,12 @@ module Gitlab return {} unless file_paths.any? results = [] - cache_key = key - highlight_diffs_renewable_expiration_enabled = Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) - expiration_period = expiration + cache_key = key # Moving out redis calls for feature flags out of redis.pipelined Gitlab::Redis::Cache.with do |redis| redis.pipelined do |pipeline| results = pipeline.hmget(cache_key, file_paths) - pipeline.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled + pipeline.expire(key, EXPIRATION) end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 7b31dd9926b..434893eab82 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -36,8 +36,14 @@ module Gitlab validate_permission!(:create_issue) + result = create_issue + issue = result[:issue] + + # issue won't be present only on unrecoverable errors + raise InvalidIssueError, result.errors.join(', ') if result.error? && issue.blank? + verify_record!( - record: create_issue, + record: issue, invalid_exception: InvalidIssueError, record_name: 'issue') end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 8e2c7559bc1..06365296a76 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -91,7 +91,7 @@ module Gitlab end def create_issue! - @issue = ::Issues::CreateService.new( + result = ::Issues::CreateService.new( project: project, current_user: User.support_bot, params: { @@ -106,7 +106,9 @@ module Gitlab spam_params: nil ).execute - raise InvalidIssueError unless @issue.persisted? + raise InvalidIssueError if result.error? + + @issue = result[:issue] begin ::Issue::Email.create!(issue: @issue, email_message_id: mail.message_id) diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb index 11a799886ab..720262816b4 100644 --- a/lib/gitlab/email/message/in_product_marketing/trial.rb +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -42,11 +42,11 @@ module Gitlab [ s_("InProductMarketing|GitLab's premium tiers are designed to make you, your team and your application more efficient and more secure with features including but not limited to:"), list([ - s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options - ]) + s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options + ]) ].join("\n"), s_('InProductMarketing|GitLab provides static application security testing (SAST), dynamic application security testing (DAST), container scanning, and dependency scanning to help you deliver secure applications along with license compliance.'), s_('InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process.') diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index d2a78b53e1f..3982a8b87fd 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -49,10 +49,10 @@ module Gitlab [ nil, list([ - s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, - s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, - s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } - ]), + s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, + s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, + s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } + ]), nil ][series] end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 34c674c3003..b1fd35184ac 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -73,8 +73,6 @@ module Gitlab # This method escapes unsupported UTF-8 characters instead of deleting them def encode_utf8_with_escaping!(message) - return encode!(message) if Feature.disabled?(:escape_gitaly_refs) - message = force_encode_utf8(message) return message if message.valid_encoding? diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index b1a9603d3a5..3c6ed696b9d 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,5 +5,9 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end + + def self.qa_user_agent + ENV['GITLAB_QA_USER_AGENT'] + end end end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index b45970cb45a..023c8ace4d9 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -43,9 +43,18 @@ module Gitlab store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectPathChangedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectArchivedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectTransferedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, + to: ::Projects::ProjectAttributesChangedEvent, + if: -> (event) { event.pages_related? } + store.subscribe ::Pages::InvalidateDomainCacheWorker, + to: ::Projects::ProjectFeaturesChangedEvent, + if: -> (event) { event.pages_related? } store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupTransferedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupPathChangedEvent store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainUpdatedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainCreatedEvent store.subscribe ::MergeRequests::CreateApprovalEventWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 75d07a36dcd..0b18a337707 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -118,7 +118,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? Gitlab::Redis::SharedState.with do |redis| - redis.exists(@redis_shared_state_key) + redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb deleted file mode 100644 index 142d0e55593..00000000000 --- a/lib/gitlab/experimentation.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -# == Experimentation -# -# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. -# Experiment options: -# - tracking_category (optional, used to set the category when tracking an experiment event) -# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout -# -# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), -# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. -# -# To enable the experiment for 10% of the time: -# -# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random` -# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` -# -# To disable the experiment: -# -# chatops: `/chatops run feature delete experiment_key_experiment_percentage` -# console: `Feature.remove(:experiment_key_experiment_percentage)` -# -# To check the current rollout percentage: -# -# chatops: `/chatops run feature get experiment_key_experiment_percentage` -# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value` -# - -# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490 -module Gitlab - module Experimentation - EXPERIMENTS = { - }.freeze - - class << self - def get_experiment(experiment_key) - return unless EXPERIMENTS.key?(experiment_key) - - ::Gitlab::Experimentation::Experiment.new(experiment_key, **EXPERIMENTS[experiment_key]) - end - - def active?(experiment_key) - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.active? - end - - def in_experiment_group?(experiment_key, subject:) - return false if subject.blank? - return false unless active?(experiment_key) - - log_invalid_rollout(experiment_key, subject) - - experiment = get_experiment(experiment_key) - return false unless experiment - - experiment.enabled_for_index?(index_for_subject(experiment, subject)) - end - - def rollout_strategy(experiment_key) - experiment = get_experiment(experiment_key) - return unless experiment - - experiment.rollout_strategy - end - - def log_invalid_rollout(experiment_key, subject) - return if valid_subject_for_rollout_strategy?(experiment_key, subject) - - logger = Gitlab::ExperimentationLogger.build - logger.warn message: 'Subject must conform to the rollout strategy', - experiment_key: experiment_key, - subject: subject.class.to_s, - rollout_strategy: rollout_strategy(experiment_key) - end - - def valid_subject_for_rollout_strategy?(experiment_key, subject) - case rollout_strategy(experiment_key) - when :user - subject.is_a?(User) - when :group - subject.is_a?(Group) - when :cookie - subject.nil? || subject.is_a?(String) - else - false - end - end - - private - - def index_for_subject(experiment, subject) - index = Zlib.crc32("#{experiment.key}#{subject_id(subject)}") - - index % 100 - end - - def subject_id(subject) - if subject.respond_to?(:to_global_id) - subject.to_global_id.to_s - elsif subject.respond_to?(:to_s) - subject.to_s - else - raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`' - end - end - end - end -end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb deleted file mode 100644 index b09d67b8d5f..00000000000 --- a/lib/gitlab/experimentation/controller_concern.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -require 'zlib' - -# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. -# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name, subject: nil)` method -# to controllers and views. It returns true when the experiment is enabled and the user is selected as part -# of the experimental group. -# -module Gitlab - module Experimentation - module ControllerConcern - include ::Gitlab::Experimentation::GroupTypes - include Gitlab::Tracking::Helpers - extend ActiveSupport::Concern - - included do - before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group - end - - def set_experimentation_subject_id_cookie - if Gitlab.com? - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } - else - # We set the cookie before, although experiments are not conducted on self managed instances. - cookies.delete(:experimentation_subject_id) - end - end - - def push_frontend_experiment(experiment_key, subject: nil) - var_name = experiment_key.to_s.camelize(:lower) - - enabled = experiment_enabled?(experiment_key, subject: subject) - - gon.push({ experiments: { var_name => enabled } }, true) - end - - def experiment_enabled?(experiment_key, subject: nil) - return true if forced_enabled?(experiment_key) - return false if dnt_enabled? - - Experimentation.log_invalid_rollout(experiment_key, subject) - - subject ||= experimentation_subject_id - - Experimentation.in_experiment_group?(experiment_key, subject: subject) - end - - def track_experiment_event(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data.merge!(user: current_user)) - end - end - - def frontend_experimentation_tracking_data(experiment_key, action, value = nil, subject: nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value, subject: subject) do |tracking_data| - gon.push(tracking_data: tracking_data) - end - end - - def record_experiment_user(experiment_key, context = {}) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && current_user - - subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user - - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) - end - - def record_experiment_group(experiment_key, group) - return if dnt_enabled? - return unless Experimentation.active?(experiment_key) && group - - variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group - variant = tracking_group(experiment_key, nil, subject: variant_subject) - - ::Experiment.add_group(experiment_key, group: group, variant: variant) - end - - def record_experiment_conversion_event(experiment_key, context = {}) - return if dnt_enabled? - return unless current_user - return unless Experimentation.active?(experiment_key) - - ::Experiment.record_conversion_event(experiment_key, current_user, context) - end - - def experiment_tracking_category_and_group(experiment_key, subject: nil) - "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group', subject: subject)}" - end - - private - - def experimentation_subject_id - cookies.signed[:experimentation_subject_id] - end - - def track_experiment_event_for(experiment_key, action, value, subject: nil) - return unless Experimentation.active?(experiment_key) - - yield experimentation_tracking_data(experiment_key, action, value, subject: subject) - end - - def experimentation_tracking_data(experiment_key, action, value, subject: nil) - { - category: tracking_category(experiment_key), - action: action, - property: tracking_group(experiment_key, "_group", subject: subject), - label: tracking_label(subject), - value: value - }.compact - end - - def tracking_category(experiment_key) - Experimentation.get_experiment(experiment_key).tracking_category - end - - def tracking_group(experiment_key, suffix = nil, subject: nil) - return unless Experimentation.active?(experiment_key) - - subject ||= experimentation_subject_id - group = experiment_enabled?(experiment_key, subject: subject) ? GROUP_EXPERIMENTAL : GROUP_CONTROL - - suffix ? "#{group}#{suffix}" : group - end - - def forced_enabled?(experiment_key) - return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s - return false if cookies[:force_experiment].blank? - - cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } - end - - def tracking_label(subject = nil) - return experimentation_subject_id if subject.blank? - - if subject.respond_to?(:to_global_id) - Digest::SHA256.hexdigest(subject.to_global_id.to_s) - else - Digest::SHA256.hexdigest(subject.to_s) - end - end - end - end -end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb deleted file mode 100644 index 0c7091d19e3..00000000000 --- a/lib/gitlab/experimentation/experiment.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Experimentation - class Experiment - FEATURE_FLAG_SUFFIX = "_experiment_percentage" - - attr_reader :key, :tracking_category, :rollout_strategy - - def initialize(key, **params) - @key = key - @tracking_category = params[:tracking_category] - @rollout_strategy = params[:rollout_strategy] || :cookie - end - - def active? - # TODO: just touch a feature flag - # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? - Feature.enabled?(feature_flag_name, type: :experiment) - - ::Gitlab.com? && experiment_percentage > 0 - end - - def enabled_for_index?(index) - return false if index.blank? - - index <= experiment_percentage - end - - private - - def experiment_percentage - feature_flag.percentage_of_time_value - end - - def feature_flag - Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet - end - - def feature_flag_name - :"#{key}#{FEATURE_FLAG_SUFFIX}" - end - end - end -end diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb deleted file mode 100644 index ba1b60d6b4c..00000000000 --- a/lib/gitlab/experimentation_logger.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class ExperimentationLogger < ::Gitlab::JsonLogger - def self.file_name_noext - 'experimentation_json' - end - end -end diff --git a/lib/gitlab/git/declared_license.rb b/lib/gitlab/git/declared_license.rb new file mode 100644 index 00000000000..bc12b1918ea --- /dev/null +++ b/lib/gitlab/git/declared_license.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Git + # DeclaredLicense is the software license declared in a LICENSE or COPYING + # file in the git repository. + class DeclaredLicense + # SPDX Identifier for the license + attr_reader :key + + # Full name of the license + attr_reader :name + + # Nickname of the license (optional, a shorter user-friendly name) + attr_reader :nickname + + # Filename of the file containing license + attr_accessor :path + + # URL that points to the LICENSE + attr_reader :url + + def initialize(key: nil, name: nil, nickname: nil, url: nil, path: nil) + @key = key + @name = name + @nickname = nickname + @url = url + @path = path + end + + def ==(other) + return unless other.is_a?(self.class) + + key == other.key + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f1cd75258be..9bbe17dcad1 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -783,10 +783,31 @@ module Gitlab end end - def license_short_name + def license(from_gitaly) wrapped_gitaly_errors do - gitaly_repository_client.license_short_name + response = gitaly_repository_client.find_license + + break nil if response.license_short_name.empty? + + if from_gitaly + break ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, + name: response.license_name, + nickname: response.license_nickname.presence, + url: response.license_url.presence, + path: response.license_path) + end + + licensee_object = Licensee::License.new(response.license_short_name) + + break nil if licensee_object.name.blank? + + licensee_object.meta.nickname = "LICENSE" if licensee_object.key == "other" + + licensee_object end + rescue Licensee::InvalidLicense => e + Gitlab::ErrorTracking.track_exception(e) + nil end def fetch_source_branch!(source_repository, source_branch, local_ref) @@ -1008,8 +1029,8 @@ module Gitlab @praefect_info_client ||= Gitlab::GitalyClient::PraefectInfoService.new(self) end - def branch_names_contains_sha(sha) - gitaly_ref_client.branch_names_contains_sha(sha) + def branch_names_contains_sha(sha, limit: 0) + gitaly_ref_client.branch_names_contains_sha(sha, limit: limit) end def tag_names_contains_sha(sha, limit: 0) diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb deleted file mode 100644 index 2228fcb886e..00000000000 --- a/lib/gitlab/git/wiki.rb +++ /dev/null @@ -1,131 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Git - class Wiki - include Gitlab::Git::WrapsGitalyErrors - - DuplicatePageError = Class.new(StandardError) - - DEFAULT_PAGINATION = Kaminari.config.default_per_page - - CommitDetails = Struct.new(:user_id, :username, :name, :email, :message) do - def to_h - { user_id: user_id, username: username, name: name, email: email, message: message } - end - end - - # GollumSlug inlines just enough knowledge from Gollum::Page to generate a - # slug, which is used when previewing pages that haven't been persisted - class GollumSlug - class << self - def cname(name, char_white_sub = '-', char_other_sub = '-') - if name.respond_to?(:gsub) - name.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub) - else - '' - end - end - - def format_to_ext(format) - format == :markdown ? "md" : format.to_s - end - - def canonicalize_filename(filename) - ::File.basename(filename, ::File.extname(filename)).tr('-', ' ') - end - - def generate(title, format) - ext = format_to_ext(format.to_sym) - name = cname(title) + '.' + ext - canonical_name = canonicalize_filename(name) - - path = - if name.include?('/') - name.sub(%r{/[^/]+$}, '/') - else - '' - end - - path + cname(canonical_name, '-', '-') - end - end - end - - attr_reader :repository - - # TODO remove argument when issue - # https://gitlab.com/gitlab-org/gitlab/-/issues/329190 - # is closed. - def self.default_ref(container = nil) - Gitlab::DefaultBranch.value(object: container) - end - - # Initialize with a Gitlab::Git::Repository instance - def initialize(repository) - @repository = repository - end - - def repository_exists? - @repository.exists? - end - - def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) - wrapped_gitaly_errors do - gitaly_list_pages( - limit: limit, - sort: sort, - direction_desc: direction_desc, - load_content: load_content - ) - end - end - - def page(title:, version: nil, dir: nil, load_content: true) - wrapped_gitaly_errors do - gitaly_find_page(title: title, version: version, dir: dir, load_content: load_content) - end - end - - def count_page_versions(page_path) - @repository.count_commits(ref: 'HEAD', path: page_path) - end - - def preview_slug(title, format) - GollumSlug.generate(title, format) - end - - private - - def gitaly_wiki_client - @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) - end - - def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) - return unless title.present? - - wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) - return unless wiki_page - - Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) - rescue GRPC::InvalidArgument - nil - end - - def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) - params = { limit: limit, sort: sort, direction_desc: direction_desc } - - gitaly_pages = - if load_content - gitaly_wiki_client.load_all_pages(**params) - else - gitaly_wiki_client.list_all_pages(**params) - end - - gitaly_pages.map do |wiki_page, version| - Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) - end - end - end - end -end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index 57b7e7d53dd..26d15daf093 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -5,22 +5,6 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data - class << self - # Abstracts away Gitlab::GitalyClient::WikiPage - def from_gitaly_wiki_page(gitaly_page, version) - new( - url_path: gitaly_page.url_path, - title: gitaly_page.title, - format: gitaly_page.format, - path: gitaly_page.path, - raw_data: gitaly_page.raw_data, - name: gitaly_page.name, - historical: gitaly_page.historical?, - version: version - ) - end - end - def initialize(hash) @url_path = hash[:url_path] @title = hash[:title] @@ -41,6 +25,11 @@ module Gitlab @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) end + + def raw_data=(data) + @raw_data = data + @text_data = @raw_data && Gitlab::EncodingHelper.encode!(@raw_data.dup) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1c5ad650678..9a3f5fb844b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -394,7 +394,7 @@ module Gitlab elsif user user.can?(:read_project, project) elsif ci? - true # allow CI (build without a user) for backwards compatibility + false end || Guest.can?(:read_project, project) end @@ -445,9 +445,6 @@ module Gitlab nil when Key actor.user - when :ci - Gitlab::AppJsonLogger.info(message: 'Actor was :ci', project_id: project.id) - nil end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index bb6bc3121bd..d2b702f3a6d 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -270,15 +270,13 @@ module Gitlab end def consume_find_local_branches_response(response) - if Feature.enabled?(:gitaly_simplify_find_local_branches_response, type: :undefined) - response.flat_map do |message| + response.flat_map do |message| + if message.local_branches.present? message.local_branches.map do |branch| target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) end - end - else - response.flat_map do |message| + else message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @repository, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 04d6f92e8d8..f11437552e1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -283,12 +283,10 @@ module Gitlab response.path.presence end - def license_short_name + def find_license request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.fast_timeout) - - response.license_short_name.presence + GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) end def calculate_checksum diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb deleted file mode 100644 index ca839b232cf..00000000000 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require 'stringio' - -module Gitlab - module GitalyClient - class WikiService - include Gitlab::EncodingHelper - - MAX_MSG_SIZE = 128.kilobytes.freeze - - def initialize(repository) - @gitaly_repo = repository.gitaly_repository - @repository = repository - end - - def write_page(name, format, content, commit_details) - request = Gitaly::WikiWritePageRequest.new( - repository: @gitaly_repo, - name: encode_binary(name), - format: format.to_s, - commit_details: gitaly_commit_details(commit_details) - ) - - strio = binary_io(content) - - enum = Enumerator.new do |y| - until strio.eof? - request.content = strio.read(MAX_MSG_SIZE) - - y.yield request - - request = Gitaly::WikiWritePageRequest.new - end - end - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout) - if error = response.duplicate_error.presence - raise Gitlab::Git::Wiki::DuplicatePageError, error - end - end - - def update_page(page_path, title, format, content, commit_details) - request = Gitaly::WikiUpdatePageRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - title: encode_binary(title), - format: format.to_s, - commit_details: gitaly_commit_details(commit_details) - ) - - strio = binary_io(content) - - enum = Enumerator.new do |y| - until strio.eof? - request.content = strio.read(MAX_MSG_SIZE) - - y.yield request - - request = Gitaly::WikiUpdatePageRequest.new - end - end - - GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) - end - - def find_page(title:, version: nil, dir: nil, load_content: true) - request = Gitaly::WikiFindPageRequest.new( - repository: @gitaly_repo, - title: encode_binary(title), - revision: encode_binary(version), - directory: encode_binary(dir), - skip_content: !load_content - ) - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) - - wiki_page_from_iterator(response) - end - - def list_all_pages(limit: 0, sort: nil, direction_desc: false) - sort_value = Gitaly::WikiListPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) - - params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } - params[:sort] = sort_value if sort_value - - request = Gitaly::WikiListPagesRequest.new(params) - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_list_pages, request, timeout: GitalyClient.medium_timeout) - stream.each_with_object([]) do |message, pages| - page = message.page - - next unless page - - wiki_page = GitalyClient::WikiPage.new(page.to_h) - version = new_wiki_page_version(page.version) - - pages << [wiki_page, version] - end - end - - def load_all_pages(limit: 0, sort: nil, direction_desc: false) - sort_value = Gitaly::WikiGetAllPagesRequest::SortBy.resolve(sort.to_s.upcase.to_sym) - - params = { repository: @gitaly_repo, limit: limit, direction_desc: direction_desc } - params[:sort] = sort_value if sort_value - - request = Gitaly::WikiGetAllPagesRequest.new(params) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) - - pages = [] - - loop do - page, version = wiki_page_from_iterator(response) { |message| message.end_of_page } - - break unless page && version - - pages << [page, version] - end - - pages - end - - private - - # If a block is given and the yielded value is truthy, iteration will be - # stopped early at that point; else the iterator is consumed entirely. - # The iterator is traversed with `next` to allow resuming the iteration. - def wiki_page_from_iterator(iterator) - wiki_page = version = nil - - while message = iterator.next - break if block_given? && yield(message) - - page = message.page - next unless page - - if wiki_page - wiki_page.raw_data << page.raw_data - else - wiki_page = GitalyClient::WikiPage.new(page.to_h) - - version = new_wiki_page_version(page.version) - end - end - - [wiki_page, version] - rescue StopIteration - [wiki_page, version] - end - - def new_wiki_page_version(version) - Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, version.commit), - version.format - ) - end - - def gitaly_commit_details(commit_details) - Gitaly::WikiCommitDetails.new( - user_id: commit_details.user_id, - user_name: encode_binary(commit_details.username), - name: encode_binary(commit_details.name), - email: encode_binary(commit_details.email), - message: encode_binary(commit_details.message) - ) - end - end - end -end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 6cff15a204f..0f89a7b6575 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -69,7 +69,7 @@ module Gitlab # # username - The username of the user. def user(username) - with_rate_limit { octokit.user(username) } + with_rate_limit { octokit.user(username).to_h } end def pull_request_reviews(repo_name, iid) @@ -88,7 +88,7 @@ module Gitlab end def pull_request(repo_name, iid) - with_rate_limit { octokit.pull_request(repo_name, iid) } + with_rate_limit { octokit.pull_request(repo_name, iid).to_h } end def labels(*args) @@ -108,7 +108,7 @@ module Gitlab end def branch_protection(repo_name, branch_name) - with_rate_limit { octokit.branch_protection(repo_name, branch_name) } + with_rate_limit { octokit.branch_protection(repo_name, branch_name).to_h } end # Fetches data from the GitHub API and yields a Page object for every page @@ -150,7 +150,7 @@ module Gitlab each_page(method, *args) do |page| page.objects.each do |object| - yield object + yield object.to_h end end end @@ -183,7 +183,7 @@ module Gitlab end def search_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" query = [query, collaborations_subquery].join(' ') if include_collaborations query = [query, organizations_subquery].join(' ') if include_orgs @@ -274,13 +274,13 @@ module Gitlab def collaborations_subquery each_object(:repos, nil, { affiliation: 'collaborator' }) - .map { |repo| "repo:#{repo.full_name}" } + .map { |repo| "repo:#{repo[:full_name]}" } .join(' ') end def organizations_subquery each_object(:organizations) - .map { |org| "org:#{org.login}" } + .map { |org| "org:#{org[:login]}" } .join(' ') end diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb new file mode 100644 index 00000000000..3a36b64a11b --- /dev/null +++ b/lib/gitlab/github_import/exceptions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Exceptions + # Sometimes it's not clear which of not implemented interfaces caused this error. + # We need custom exception to be able to add text that gives extra context. + NotImplementedError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/base_importer.rb b/lib/gitlab/github_import/importer/attachments/base_importer.rb new file mode 100644 index 00000000000..eaff99aed43 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/base_importer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class BaseImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + collection.each_batch(of: BATCH_SIZE, column: ordering_column) do |batch| + batch.each do |record| + next if already_imported?(record) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield record + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(record) + end + end + end + + def representation_class + Representation::NoteText + end + + def importer_class + NoteAttachmentsImporter + end + + private + + def collection + raise Gitlab::GithubImport::Exceptions::NotImplementedError, '#collection' + end + + def ordering_column + :id + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb new file mode 100644 index 00000000000..090bfb4a098 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class IssuesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportIssueWorker + end + + def collection_method + :issue_attachments + end + + def object_type + :issue_attachment + end + + def id_for_already_imported_cache(issue) + issue.id + end + + private + + def collection + project.issues.select(:id, :description) + end + + def ordering_column + :iid + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb new file mode 100644 index 00000000000..f41071b1785 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class MergeRequestsImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportMergeRequestWorker + end + + def collection_method + :merge_request_attachments + end + + def object_type + :merge_request_attachment + end + + def id_for_already_imported_cache(merge_request) + merge_request.id + end + + private + + def collection + project.merge_requests.select(:id, :description) + end + + def ordering_column + :iid + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb new file mode 100644 index 00000000000..aa38a7a3a3f --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class NotesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportNoteWorker + end + + def collection_method + :note_attachments + end + + def object_type + :note_attachment + end + + def id_for_already_imported_cache(note) + note.id + end + + private + + # TODO: exclude :system, :noteable_type from select after removing override Note#note method + # https://gitlab.com/gitlab-org/gitlab/-/issues/369923 + def collection + project.notes.user.select(:id, :note, :system, :noteable_type) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb new file mode 100644 index 00000000000..feaa69eff71 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class ReleasesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportReleaseWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + private + + def collection + project.releases.select(:id, :description) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb index 49cbc8f7a42..92f26692a05 100644 --- a/lib/gitlab/github_import/importer/diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(note) - note.id + note[:id] end end end diff --git a/lib/gitlab/github_import/importer/issue_events_importer.rb b/lib/gitlab/github_import/importer/issue_events_importer.rb index 71dd99f91f9..a1c706c5d78 100644 --- a/lib/gitlab/github_import/importer/issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/issue_events_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(event) - event.id + event[:id] end end end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index 21d9ce8cd2d..3d6f15fc2bc 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -33,13 +33,17 @@ module Gitlab end def id_for_already_imported_cache(issue) - issue.number + issue[:number] end def collection_options { state: 'all', sort: 'created', direction: 'asc' } end + def increment_object_counter?(object) + object[:pull_request].nil? + end + private def additional_object_data diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 7293de56a9a..9a011f17a18 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -22,7 +22,7 @@ module Gitlab end def already_imported?(label) - existing_labels.include?(label.name) + existing_labels.include?(label[:name]) end def build_labels_cache @@ -33,8 +33,8 @@ module Gitlab time = Time.zone.now { - title: label.name, - color: '#' + label.color, + title: label[:name], + color: '#' + label[:color], project_id: project.id, type: 'ProjectLabel', created_at: time, diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index d11b151bbe2..1a3a54d0053 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -22,7 +22,7 @@ module Gitlab end def already_imported?(milestone) - existing_milestones.include?(milestone.number) + existing_milestones.include?(milestone[:number]) end def build_milestones_cache @@ -31,19 +31,19 @@ module Gitlab def build(milestone) { - iid: milestone.number, - title: milestone.title, - description: milestone.description, + iid: milestone[:number], + title: milestone[:title], + description: milestone[:description], project_id: project.id, state: state_for(milestone), - due_date: milestone.due_on&.to_date, - created_at: milestone.created_at, - updated_at: milestone.updated_at + due_date: milestone[:due_on]&.to_date, + created_at: milestone[:created_at], + updated_at: milestone[:updated_at] } end def state_for(milestone) - milestone.state == 'open' ? :active : :closed + milestone[:state] == 'open' ? :active : :closed end def each_milestone diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb new file mode 100644 index 00000000000..9901c9e76f5 --- /dev/null +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class NoteAttachmentsImporter + attr_reader :note_text, :project + + # note_text - An instance of `NoteText`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(note_text, project, _client = nil) + @note_text = note_text + @project = project + end + + def execute + attachments = MarkdownText.fetch_attachments(note_text.text) + return if attachments.blank? + + new_text = attachments.reduce(note_text.text) do |text, attachment| + new_url = download_attachment(attachment) + text.gsub(attachment.url, new_url) + end + + update_note_record(new_text) + end + + private + + # in: an instance of Gitlab::GithubImport::Markdown::Attachment + # out: gitlab attachment markdown url + def download_attachment(attachment) + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(attachment.url) + file = downloader.perform + uploader = UploadService.new(project, file, FileUploader).execute + uploader.to_h[:url] + ensure + downloader&.delete + end + + def update_note_record(text) + case note_text.record_type + when ::Release.name + ::Release.find(note_text.record_db_id).update_column(:description, text) + when ::Issue.name + ::Issue.find(note_text.record_db_id).update_column(:description, text) + when ::MergeRequest.name + ::MergeRequest.find(note_text.record_db_id).update_column(:description, text) + when ::Note.name + ::Note.find(note_text.record_db_id).update_column(:note, text) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 1410006af26..69b7b2c2a38 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -36,6 +36,9 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. + # Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb index ca1d7d60515..4c2b87a8c5e 100644 --- a/lib/gitlab/github_import/importer/notes_importer.rb +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -27,7 +27,7 @@ module Gitlab end def id_for_already_imported_cache(note) - note.id + note[:id] end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 16215fdce8e..21075e21e1d 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -6,6 +6,10 @@ module Gitlab class ProtectedBranchImporter attr_reader :protected_branch, :project, :client + # By default on GitHub, both developers and maintainers can merge + # a PR into the protected branch + GITHUB_DEFAULT_MERGE_ACCESS_LEVEL = Gitlab::Access::DEVELOPER + # protected_branch - An instance of # `Gitlab::GithubImport::Representation::ProtectedBranch`. # project - An instance of `Project` @@ -22,6 +26,8 @@ module Gitlab ProtectedBranches::CreateService .new(project, project.creator, params) .execute(skip_authorization: true) + + update_project_settings if default_branch? end private @@ -29,8 +35,8 @@ module Gitlab def params { name: protected_branch.id, - push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], - merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + push_access_levels_attributes: [{ access_level: push_access_level }], + merge_access_levels_attributes: [{ access_level: merge_access_level }], allow_force_push: allow_force_push? } end @@ -42,6 +48,95 @@ module Gitlab protected_branch.allow_force_pushes end end + + def default_branch? + protected_branch.id == project.default_branch + end + + def update_project_settings + update_setting_for_only_allow_merge_if_all_discussions_are_resolved + update_project_push_rule + end + + def update_setting_for_only_allow_merge_if_all_discussions_are_resolved + return unless protected_branch.required_conversation_resolution + + project.update(only_allow_merge_if_all_discussions_are_resolved: true) + end + + def update_project_push_rule + return unless project.licensed_feature_available?(:push_rules) + return unless protected_branch.required_signatures + + push_rule = project.push_rule || project.build_push_rule + push_rule.update!(reject_unsigned_commits: true) + project.project_setting.update!(push_rule_id: push_rule.id) + end + + def push_access_level + if protected_branch.required_pull_request_reviews + Gitlab::Access::NO_ACCESS + else + gitlab_access_level_for(:push) + end + end + + # Gets the strictest merge_access_level between GitHub and GitLab + def merge_access_level + gitlab_access = gitlab_access_level_for(:merge) + + return gitlab_access if gitlab_access == Gitlab::Access::NO_ACCESS + + [gitlab_access, GITHUB_DEFAULT_MERGE_ACCESS_LEVEL].max + end + + # action - :push/:merge + def gitlab_access_level_for(action) + if default_branch? + action == :push ? default_branch_push_access_level : default_branch_merge_access_level + elsif protected_on_gitlab? + non_default_branch_access_level_for(action) + else + gitlab_default_access_level_for(action) + end + end + + def default_branch_push_access_level + if default_branch_protection.developer_can_push? + Gitlab::Access::DEVELOPER + else + gitlab_default_access_level_for(:push) + end + end + + def default_branch_merge_access_level + if default_branch_protection.developer_can_merge? + Gitlab::Access::DEVELOPER + else + gitlab_default_access_level_for(:merge) + end + end + + def default_branch_protection + Gitlab::Access::BranchProtection.new(project.namespace.default_branch_protection) + end + + def protected_on_gitlab? + ProtectedBranch.protected?(project, protected_branch.id) + end + + def non_default_branch_access_level_for(action) + access_level = ProtectedBranch.access_levels_for_ref(protected_branch.id, action: action) + .find(&:role?)&.access_level + + access_level || gitlab_default_access_level_for(action) + end + + def gitlab_default_access_level_for(action) + return ProtectedBranch::PushAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL if action == :push + + ProtectedBranch::MergeAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL + end end end end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb index b5be823d5ab..4372477f55d 100644 --- a/lib/gitlab/github_import/importer/protected_branches_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -11,9 +11,9 @@ module Gitlab def each_object_to_import repo = project.import_source - protected_branches = client.branches(repo).select { |branch| branch.protection&.enabled } + protected_branches = client.branches(repo).select { |branch| branch.dig(:protection, :enabled) } protected_branches.each do |protected_branch| - object = client.branch_protection(repo, protected_branch.name) + object = client.branch_protection(repo, protected_branch[:name]) next if object.nil? || already_imported?(object) yield object @@ -44,7 +44,7 @@ module Gitlab end def id_for_already_imported_cache(protected_branch) - protected_branch.name + protected_branch[:name] end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 5d291d9d723..16541c90002 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -19,7 +19,7 @@ module Gitlab end def id_for_already_imported_cache(pr) - pr.number + pr[:number] end def object_type @@ -55,11 +55,11 @@ module Gitlab def update_repository?(pr) last_update = project.last_repository_updated_at || project.created_at - return false if pr.updated_at < last_update + return false if pr[:updated_at] < last_update # PRs may be updated without there actually being new commits, thus we # check to make sure we only re-fetch if truly necessary. - !(commit_exists?(pr.head.sha) && commit_exists?(pr.base.sha)) + !(commit_exists?(pr.dig(:head, :sha)) && commit_exists?(pr.dig(:base, :sha))) end def commit_exists?(sha) diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 5e55d09fe3d..543c29a21a0 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -34,7 +34,7 @@ module Gitlab end def id_for_already_imported_cache(review) - review.id + review[:id] end # The worker can be interrupted, by rate limit for instance, @@ -48,11 +48,13 @@ module Gitlab def each_object_to_import(&block) each_review_page do |page, merge_request| page.objects.each do |review| + review = review.to_h + next if already_imported?(review) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - review.merge_request_id = merge_request.id + review[:merge_request_id] = merge_request.id yield(review) mark_as_imported(review) diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/release_attachments_importer.rb deleted file mode 100644 index 6419851623c..00000000000 --- a/lib/gitlab/github_import/importer/release_attachments_importer.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class ReleaseAttachmentsImporter - attr_reader :release_db_id, :release_description, :project - - # release - An instance of `ReleaseAttachments`. - # project - An instance of `Project`. - # client - An instance of `Gitlab::GithubImport::Client`. - def initialize(release_attachments, project, _client = nil) - @release_db_id = release_attachments.release_db_id - @release_description = release_attachments.description - @project = project - end - - def execute - attachment_urls = MarkdownText.fetch_attachment_urls(release_description) - new_description = attachment_urls.reduce(release_description) do |description, url| - new_url = download_attachment(url) - description.gsub(url, new_url) - end - - Release.find(release_db_id).update_column(:description, new_description) - end - - private - - # in: github attachment markdown url - # out: gitlab attachment markdown url - def download_attachment(markdown_url) - url = extract_url_from_markdown(markdown_url) - name_prefix = extract_name_from_markdown(markdown_url) - - downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(url) - file = downloader.perform - uploader = UploadService.new(project, file, FileUploader).execute - "#{name_prefix}(#{uploader.to_h[:url]})" - ensure - downloader&.delete - end - - # in: "![image-icon](https://user-images.githubusercontent.com/..)" - # out: https://user-images.githubusercontent.com/.. - def extract_url_from_markdown(text) - text.match(%r{https://.*\)$}).to_a[0].chop - end - - # in: "![image-icon](https://user-images.githubusercontent.com/..)" - # out: ![image-icon] - def extract_name_from_markdown(text) - text.match(%r{^!?\[.*\]}).to_a[0] - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb deleted file mode 100644 index 7221c802d83..00000000000 --- a/lib/gitlab/github_import/importer/releases_attachments_importer.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class ReleasesAttachmentsImporter - include ParallelScheduling - - BATCH_SIZE = 100 - - # The method that will be called for traversing through all the objects to - # import, yielding them to the supplied block. - def each_object_to_import - project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| - batch.each do |release| - next if already_imported?(release) - - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - yield release - - # We mark the object as imported immediately so we don't end up - # scheduling it multiple times. - mark_as_imported(release) - end - end - end - - def representation_class - Representation::ReleaseAttachments - end - - def importer_class - ReleaseAttachmentsImporter - end - - def sidekiq_worker_class - ImportReleaseAttachmentsWorker - end - - def collection_method - :release_attachments - end - - def object_type - :release_attachment - end - - def id_for_already_imported_cache(release) - release.id - end - - def object_representation(object) - representation_class.from_db_record(object) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 51d364772d2..fe6da30bbf8 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -12,6 +12,9 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Note: if you're going to replace `legacy_bulk_insert` with something that triggers callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute bulk_insert(Release, build_releases) end @@ -21,21 +24,21 @@ module Gitlab end def already_imported?(release) - existing_tags.include?(release.tag_name) || release.tag_name.nil? + existing_tags.include?(release[:tag_name]) || release[:tag_name].nil? end def build(release) - existing_tags.add(release.tag_name) + existing_tags.add(release[:tag_name]) { - name: release.name, - tag: release.tag_name, + name: release[:name], + tag: release[:tag_name], author_id: fetch_author_id(release), description: description_for(release), - created_at: release.created_at, - updated_at: release.created_at, + created_at: release[:created_at], + updated_at: release[:created_at], # Draft releases will have a null published_at - released_at: release.published_at || Time.current, + released_at: release[:published_at] || Time.current, project_id: project.id } end @@ -45,7 +48,7 @@ module Gitlab end def description_for(release) - release.body.presence || "Release for tag #{release.tag_name}" + release[:body].presence || "Release for tag #{release[:tag_name]}" end def object_type diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8a9ddfc6ec0..4090555c85e 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -22,13 +22,15 @@ module Gitlab # To make it possible to identify issue in separated worker we need to patch # Sawyer instances here with issue number def each_associated(parent_record, associated) + associated = associated.to_h + compose_associated_id!(parent_record, associated) return if already_imported?(associated) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) pull_request = parent_record.is_a? MergeRequest - associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } + associated[:issue] = { number: parent_record.iid, pull_request: pull_request } yield(associated) mark_as_imported(associated) @@ -78,7 +80,7 @@ module Gitlab end def id_for_already_imported_cache(event) - event.id + event[:id] end def collection_options @@ -87,9 +89,9 @@ module Gitlab # Cross-referenced events on Github doesn't have id. def compose_associated_id!(issuable, event) - return if event.event != 'cross-referenced' + return if event[:event] != 'cross-referenced' - event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" + event[:id] = "cross-reference##{issuable.iid}-in-#{event.dig(:source, :issue, :id)}" end end end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index e7a1b7b3368..b960df581e4 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -80,12 +80,16 @@ module Gitlab end def timeout - if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) + if import_settings.enabled?(:single_endpoint_notes_import) Gitlab::Cache::Import::Caching::LONGER_TIMEOUT else Gitlab::Cache::Import::Caching::TIMEOUT end end + + def import_settings + ::Gitlab::GithubImport::Settings.new(project) + end end end end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb new file mode 100644 index 00000000000..a5cf5ffa60e --- /dev/null +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Markdown + class Attachment + MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze + DOC_TYPES = %w[ + csv docx fodg fodp fods fodt gz log md odf odg odp ods + odt pdf pptx tgz txt xls xlsx zip + ].freeze + + class << self + # markdown_node - CommonMarker::Node + def from_markdown(markdown_node) + case markdown_node.type + when :html, :inline_html + from_inline_html(markdown_node) + when :image + from_markdown_image(markdown_node) + when :link + from_markdown_link(markdown_node) + end + end + + private + + def from_markdown_image(markdown_node) + url = markdown_node.url + + return unless github_url?(url, media: true) + return unless whitelisted_type?(url, media: true) + + new(markdown_node.to_plaintext.strip, url) + end + + def from_markdown_link(markdown_node) + url = markdown_node.url + + return unless github_url?(url, docs: true) + return unless whitelisted_type?(url, docs: true) + + new(markdown_node.to_plaintext.strip, url) + end + + def from_inline_html(markdown_node) + img = Nokogiri::HTML.parse(markdown_node.string_content).xpath('//img')[0] + + return unless img + return unless github_url?(img[:src], media: true) + return unless whitelisted_type?(img[:src], media: true) + + new(img[:alt], img[:src]) + end + + def github_url?(url, docs: false, media: false) + if media + url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + elsif docs + url.start_with?(::Gitlab::GithubImport::MarkdownText.github_url) + end + end + + def whitelisted_type?(url, docs: false, media: false) + if media + MEDIA_TYPES.any? { |type| url.end_with?(type) } + elsif docs + DOC_TYPES.any? { |type| url.end_with?(type) } + end + end + end + + attr_reader :name, :url + + def initialize(name, url) + @name = name + @url = url + end + + def inspect + "<#{self.class.name}: { name: #{name}, url: #{url} }>" + end + end + end + end +end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index bf2856bc77f..2424b3e8c1f 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -8,23 +8,12 @@ module Gitlab class MarkdownText include Gitlab::EncodingHelper - ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' - PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' - - MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze - DOC_TYPES = %w[ - csv docx fodg fodp fods fodt gz log md odf odg odp ods - odt pdf pptx tgz txt xls xlsx zip - ].freeze - ALL_TYPES = (MEDIA_TYPES + DOC_TYPES).freeze - # On github.com we have base url for docs and CDN url for media. # On github EE as far as we can know there is no CDN urls and media is placed on base url. - # To no escape the escaping symbol we use single quotes instead of double with interpolation. - # rubocop:disable Style/StringConcatenation - CDN_URL_MATCHER = '(!\[.+\]\(%{github_media_cdn}/\d+/(\w|-)+\.(' + MEDIA_TYPES.join('|') + ')\))' - BASE_URL_MATCHER = '(\[.+\]\(%{github_url}/.+/.+/files/\d+/.+\.(' + ALL_TYPES.join('|') + ')\))' - # rubocop:enable Style/StringConcatenation + GITHUB_MEDIA_CDN = 'https://user-images.githubusercontent.com' + + ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' + PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' class << self def format(*args) @@ -42,20 +31,6 @@ module Gitlab .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) end - def fetch_attachment_urls(text) - cdn_url_matcher = CDN_URL_MATCHER % { github_media_cdn: Regexp.escape(github_media_cdn) } - doc_url_matcher = BASE_URL_MATCHER % { github_url: Regexp.escape(github_url) } - - text.scan(Regexp.new(cdn_url_matcher)).map(&:first) + - text.scan(Regexp.new(doc_url_matcher)).map(&:first) - end - - private - - def github_media_cdn - 'https://user-images.githubusercontent.com' - end - # Returns github domain without slash in the end def github_url oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} @@ -63,6 +38,23 @@ module Gitlab url = url.chop if url.end_with?('/') url end + + def fetch_attachments(text) + attachments = [] + doc = CommonMarker.render_doc(text) + + doc.walk do |node| + attachment = extract_attachment(node) + attachments << attachment if attachment + end + attachments + end + + private + + def extract_attachment(node) + ::Gitlab::GithubImport::Markdown::Attachment.from_markdown(node) + end end # text - The Markdown text as a String. diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index bf5046de36c..03aa02fb659 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -125,9 +125,13 @@ module Gitlab next unless page_counter.set(page.number) page.objects.each do |object| + object = object.to_h + next if already_imported?(object) - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + if increment_object_counter?(object) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + end yield object @@ -138,6 +142,10 @@ module Gitlab end end + def increment_object_counter?(_object) + true + end + # Returns true if the given object has already been imported, false # otherwise. # diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 64aa6ea5cb4..f3be90834c7 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -19,33 +19,33 @@ module Gitlab # Builds a diff note from a GitHub API response. # - # note - An instance of `Sawyer::Resource` containing the note details. + # note - An instance of `Hash` containing the note details. def self.from_api_response(note, additional_data = {}) - matches = note.html_url.match(NOTEABLE_ID_REGEX) + matches = note[:html_url].match(NOTEABLE_ID_REGEX) unless matches raise( ArgumentError, - "The note URL #{note.html_url.inspect} is not supported" + "The note URL #{note[:html_url].inspect} is not supported" ) end - user = Representation::User.from_api_response(note.user) if note.user + user = Representation::User.from_api_response(note[:user]) if note[:user] hash = { noteable_id: matches[:iid].to_i, - file_path: note.path, - commit_id: note.commit_id, - original_commit_id: note.original_commit_id, - diff_hunk: note.diff_hunk, + file_path: note[:path], + commit_id: note[:commit_id], + original_commit_id: note[:original_commit_id], + diff_hunk: note[:diff_hunk], author: user, - note: note.body, - created_at: note.created_at, - updated_at: note.updated_at, - note_id: note.id, - end_line: note.line, - start_line: note.start_line, - side: note.side, - in_reply_to_id: note.in_reply_to_id + note: note[:body], + created_at: note[:created_at], + updated_at: note[:updated_at], + note_id: note[:id], + end_line: note[:line], + start_line: note[:start_line], + side: note[:side], + in_reply_to_id: note[:in_reply_to_id] } new(hash) diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 9d457ec1c2f..e878aeaf3b9 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -15,28 +15,28 @@ module Gitlab # Builds an issue from a GitHub API response. # - # issue - An instance of `Sawyer::Resource` containing the issue + # issue - An instance of `Hash` containing the issue # details. def self.from_api_response(issue, additional_data = {}) user = - if issue.user - Representation::User.from_api_response(issue.user) + if issue[:user] + Representation::User.from_api_response(issue[:user]) end hash = { - iid: issue.number, - title: issue.title, - description: issue.body, - milestone_number: issue.milestone&.number, - state: issue.state == 'open' ? :opened : :closed, - assignees: issue.assignees.map do |u| + iid: issue[:number], + title: issue[:title], + description: issue[:body], + milestone_number: issue.dig(:milestone, :number), + state: issue[:state] == 'open' ? :opened : :closed, + assignees: issue[:assignees].map do |u| Representation::User.from_api_response(u) end, - label_names: issue.labels.map(&:name), + label_names: issue[:labels].map { _1[:name] }, author: user, - created_at: issue.created_at, - updated_at: issue.updated_at, - pull_request: issue.pull_request ? true : false, + created_at: issue[:created_at], + updated_at: issue[:updated_at], + pull_request: issue[:pull_request] ? true : false, work_item_type_id: additional_data[:work_item_type_id] } diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 89271a7dcd6..39a23c016ce 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -34,23 +34,23 @@ module Gitlab class << self # Builds an event from a GitHub API response. # - # event - An instance of `Sawyer::Resource` containing the event details. + # event - An instance of `Hash` containing the event details. def from_api_response(event, additional_data = {}) new( - id: event.id, - actor: user_representation(event.actor), - event: event.event, - commit_id: event.commit_id, - label_title: event.label && event.label[:name], - old_title: event.rename && event.rename[:from], - new_title: event.rename && event.rename[:to], - milestone_title: event.milestone && event.milestone[:title], - issue: event.issue&.to_h&.symbolize_keys, - source: event.source, - assignee: user_representation(event.assignee), - requested_reviewer: user_representation(event.requested_reviewer), - review_requester: user_representation(event.review_requester), - created_at: event.created_at + id: event[:id], + actor: user_representation(event[:actor]), + event: event[:event], + commit_id: event[:commit_id], + label_title: event.dig(:label, :name), + old_title: event.dig(:rename, :from), + new_title: event.dig(:rename, :to), + milestone_title: event.dig(:milestone, :title), + issue: event[:issue], + source: event[:source], + assignee: user_representation(event[:assignee]), + requested_reviewer: user_representation(event[:requested_reviewer]), + review_requester: user_representation(event[:review_requester]), + created_at: event[:created_at] ) end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index ae56c370b19..14379e8a4e9 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -16,14 +16,14 @@ module Gitlab # Builds a note from a GitHub API response. # - # note - An instance of `Sawyer::Resource` containing the note details. + # note - An instance of `Hash` containing the note details. def self.from_api_response(note, additional_data = {}) - matches = note.html_url.match(NOTEABLE_TYPE_REGEX) + matches = note[:html_url].match(NOTEABLE_TYPE_REGEX) if !matches || !matches[:type] raise( ArgumentError, - "The note URL #{note.html_url.inspect} is not supported" + "The note URL #{note[:html_url].inspect} is not supported" ) end @@ -34,15 +34,15 @@ module Gitlab 'Issue' end - user = Representation::User.from_api_response(note.user) if note.user + user = Representation::User.from_api_response(note[:user]) if note[:user] hash = { noteable_type: noteable_type, noteable_id: matches[:iid].to_i, author: user, - note: note.body, - created_at: note.created_at, - updated_at: note.updated_at, - note_id: note.id + note: note[:body], + created_at: note[:created_at], + updated_at: note[:updated_at], + note_id: note[:id] } new(hash) diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb new file mode 100644 index 00000000000..505d7d805d3 --- /dev/null +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# This class only partly represents MODELS_ALLOWLIST records from DB and +# is used to connect ReleasesAttachmentsImporter, NotesAttachmentsImporter etc. +# with NoteAttachmentsImporter without modifying ObjectImporter a lot. +# Attachments are inside release's `description`. +module Gitlab + module GithubImport + module Representation + class NoteText + include ToHash + include ExposeAttribute + + MODELS_ALLOWLIST = [::Release, ::Note, ::Issue, ::MergeRequest].freeze + ModelNotSupported = Class.new(StandardError) + + attr_reader :attributes + + expose_attribute :record_db_id, :record_type, :text + + class << self + # Builds a note text representation from DB record of Note or Release. + # + # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model + def from_db_record(record) + check_record_class!(record) + + record_type = record.class.name + # only column for note is different along MODELS_ALLOWLIST + text = record.is_a?(::Note) ? record.note : record.description + new( + record_db_id: record.id, + record_type: record_type, + text: text + ) + end + + def from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + private + + def check_record_class!(record) + raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class) + end + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { db_id: record_db_id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index b80b7cf1076..07a607ae70d 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -9,18 +9,22 @@ module Gitlab attr_reader :attributes - expose_attribute :id, :allow_force_pushes + expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, + :required_pull_request_reviews # Builds a Branch Protection info from a GitHub API response. # Resource structure details: # https://docs.github.com/en/rest/branches/branch-protection#get-branch-protection - # branch_protection - An instance of `Sawyer::Resource` containing the protection details. + # branch_protection - An instance of `Hash` containing the protection details. def self.from_api_response(branch_protection, _additional_object_data = {}) - branch_name = branch_protection.url.match(%r{/branches/(\S{1,255})/protection$})[1] + branch_name = branch_protection[:url].match(%r{/branches/(\S{1,255})/protection$})[1] hash = { id: branch_name, - allow_force_pushes: branch_protection.allow_force_pushes.enabled + allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), + required_conversation_resolution: branch_protection.dig(:required_conversation_resolution, :enabled), + required_signatures: branch_protection.dig(:required_signatures, :enabled), + required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present? } new(hash) diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 2adac2af502..4b8ae1f8eab 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -17,30 +17,30 @@ module Gitlab # Builds a PR from a GitHub API response. # - # issue - An instance of `Sawyer::Resource` containing the PR details. + # issue - An instance of `Hash` containing the PR details. def self.from_api_response(pr, additional_data = {}) - assignee = Representation::User.from_api_response(pr.assignee) if pr.assignee - user = Representation::User.from_api_response(pr.user) if pr.user - merged_by = Representation::User.from_api_response(pr.merged_by) if pr.merged_by + assignee = Representation::User.from_api_response(pr[:assignee]) if pr[:assignee] + user = Representation::User.from_api_response(pr[:user]) if pr[:user] + merged_by = Representation::User.from_api_response(pr[:merged_by]) if pr[:merged_by] hash = { - iid: pr.number, - title: pr.title, - description: pr.body, - source_branch: pr.head.ref, - target_branch: pr.base.ref, - source_branch_sha: pr.head.sha, - target_branch_sha: pr.base.sha, - source_repository_id: pr.head&.repo&.id, - target_repository_id: pr.base&.repo&.id, - source_repository_owner: pr.head&.user&.login, - state: pr.state == 'open' ? :opened : :closed, - milestone_number: pr.milestone&.number, + iid: pr[:number], + title: pr[:title], + description: pr[:body], + source_branch: pr.dig(:head, :ref), + target_branch: pr.dig(:base, :ref), + source_branch_sha: pr.dig(:head, :sha), + target_branch_sha: pr.dig(:base, :sha), + source_repository_id: pr.dig(:head, :repo, :id), + target_repository_id: pr.dig(:base, :repo, :id), + source_repository_owner: pr.dig(:head, :user, :login), + state: pr[:state] == 'open' ? :opened : :closed, + milestone_number: pr.dig(:milestone, :number), author: user, assignee: assignee, - created_at: pr.created_at, - updated_at: pr.updated_at, - merged_at: pr.merged_at, + created_at: pr[:created_at], + updated_at: pr[:updated_at], + merged_at: pr[:merged_at], merged_by: merged_by } diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 8a7ecf0c588..8fb57ae89a4 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -11,16 +11,19 @@ module Gitlab expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id + # Builds a PullRequestReview from a GitHub API response. + # + # review - An instance of `Hash` containing the note details. def self.from_api_response(review, additional_data = {}) - user = Representation::User.from_api_response(review.user) if review.user + user = Representation::User.from_api_response(review[:user]) if review[:user] new( - merge_request_id: review.merge_request_id, + merge_request_id: review[:merge_request_id], author: user, - note: review.body, - review_type: review.state, - submitted_at: review.submitted_at, - review_id: review.id + note: review[:body], + review_type: review[:state], + submitted_at: review[:submitted_at], + review_id: review[:id] ) end diff --git a/lib/gitlab/github_import/representation/release_attachments.rb b/lib/gitlab/github_import/representation/release_attachments.rb deleted file mode 100644 index fd272be2405..00000000000 --- a/lib/gitlab/github_import/representation/release_attachments.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -# This class only partly represents Release record from DB and -# is used to connect ReleasesAttachmentsImporter with ReleaseAttachmentsImporter -# without modifying ObjectImporter a lot. -# Attachments are inside release's `description`. -module Gitlab - module GithubImport - module Representation - class ReleaseAttachments - include ToHash - include ExposeAttribute - - attr_reader :attributes - - expose_attribute :release_db_id, :description - - # Builds a event from a GitHub API response. - # - # release - An instance of `Release` model. - def self.from_db_record(release) - new( - release_db_id: release.id, - description: release.description - ) - end - - def self.from_json_hash(raw_hash) - new Representation.symbolize_hash(raw_hash) - end - - # attributes - A Hash containing the event details. The keys of this - # Hash (and any nested hashes) must be symbols. - def initialize(attributes) - @attributes = attributes - end - - def github_identifiers - { db_id: release_db_id } - end - end - end - end -end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index 4ef916cc41c..02cbe037384 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -13,11 +13,11 @@ module Gitlab # Builds a user from a GitHub API response. # - # user - An instance of `Sawyer::Resource` containing the user details. + # user - An instance of `Hash` containing the user details. def self.from_api_response(user, additional_data = {}) new( - id: user.id, - login: user.login + id: user[:id], + login: user[:login] ) end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb new file mode 100644 index 00000000000..77288b9fb98 --- /dev/null +++ b/lib/gitlab/github_import/settings.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class Settings + OPTIONAL_STAGES = { + single_endpoint_issue_events_import: { + label: 'Import issue and pull request events', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + For example, opened or closed, renamed, and labeled or unlabeled. + Time required to import these events depends on how many issues or pull requests your project has. + TEXT + }, + single_endpoint_notes_import: { + label: 'Use alternative comments import method', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + The default method can skip some comments in large projects because of limitations of the GitHub API. + TEXT + }, + attachments_import: { + label: 'Import Markdown attachments', + details: <<-TEXT.split("\n").map(&:strip).join(' ') + Import Markdown attachments from repository comments, release posts, issue descriptions, + and pull request descriptions. These can include images, text, or binary attachments. + If not imported, links in Markdown to attachments break after you remove the attachments from GitHub. + TEXT + } + }.freeze + + def self.stages_array + OPTIONAL_STAGES.map do |stage_name, data| + { + name: stage_name.to_s, + label: s_(format("GitHubImport|%{text}", text: data[:label])), + details: s_(format("GitHubImport|%{text}", text: data[:details])) + } + end + end + + def initialize(project) + @project = project + end + + def write(user_settings) + user_settings = user_settings.to_h.with_indifferent_access + + optional_stages = fetch_stages_from_params(user_settings) + import_data = project.create_or_update_import_data(data: { optional_stages: optional_stages }) + import_data.save! + end + + def enabled?(stage_name) + project.import_data&.data&.dig('optional_stages', stage_name.to_s) || false + end + + def disabled?(stage_name) + !enabled?(stage_name) + end + + private + + attr_reader :project + + def fetch_stages_from_params(user_settings) + OPTIONAL_STAGES.keys.to_h do |stage_name| + enabled = Gitlab::Utils.to_boolean(user_settings[stage_name], default: false) + [stage_name, enabled] + end + end + end + end +end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index aea4059dfbc..3584288da57 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -4,10 +4,10 @@ # - SingleEndpointDiffNotesImporter # - SingleEndpointIssueNotesImporter # - SingleEndpointMergeRequestNotesImporter -# if `github_importer_single_endpoint_notes_import` feature flag is on. +# if enabled by Gitlab::GithubImport::Settings # # - SingleEndpointIssueEventsImporter -# if `github_importer_issue_events_import` feature flag is on. +# if enabled by Gitlab::GithubImport::Settings # # Fetches associated objects page by page to each item of parent collection. # Currently `associated` is note or event. @@ -32,7 +32,7 @@ module Gitlab end def id_for_already_imported_cache(associated) - associated.id + associated[:id] end def parent_collection @@ -54,6 +54,8 @@ module Gitlab # in Github API response object. For example: # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26 def each_associated(_parent_record, associated) + associated = associated.to_h + return if already_imported?(associated) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 1feb0d450f0..b8751def08f 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -39,18 +39,19 @@ module Gitlab # # If the object has no author ID we'll use the ID of the GitLab ghost # user. + # object - An instance of `Hash` or a `Github::Representer` def author_id_for(object, author_key: :author) user_info = case author_key when :actor - object&.actor + object[:actor] when :assignee - object&.assignee + object[:assignee] when :requested_reviewer - object&.requested_reviewer + object[:requested_reviewer] when :review_requester - object&.review_requester + object[:review_requester] else - object&.author + object ? object[:author] : nil end id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id @@ -64,14 +65,14 @@ module Gitlab # Returns the GitLab user ID of an issuable's assignee. def assignee_id_for(issuable) - user_id_for(issuable.assignee) if issuable.assignee + user_id_for(issuable[:assignee]) if issuable[:assignee] end # Returns the GitLab user ID for a GitHub user. # - # user - An instance of `Gitlab::GithubImport::Representation::User`. + # user - An instance of `Gitlab::GithubImport::Representation::User` or `Hash`. def user_id_for(user) - find(user.id, user.login) if user.present? + find(user[:id], user[:login]) if user.present? end # Returns the GitLab ID for the given GitHub ID or username. @@ -114,7 +115,7 @@ module Gitlab unless email user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user.email, timeout: timeout(user.email)) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user end email diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 08a614edb4b..bdb7484f3d6 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -15,6 +15,7 @@ module Gitlab gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.markdown_surround_selection = current_user&.markdown_surround_selection + gon.markdown_automatic_lists = current_user&.markdown_automatic_lists if Gitlab.config.sentry.enabled gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn @@ -32,6 +33,7 @@ module Gitlab gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') + gon.gridstack_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/gridstack.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors @@ -55,7 +57,7 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) - push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) + push_frontend_feature_flag(:integration_slack_app_notifications) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 987a5e7b74b..eca4d42fb9a 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -151,7 +151,7 @@ module Gitlab def limit_value # note: only first _or_ last can be specified, not both - @limit_value ||= [first, last, max_page_size, GitlabSchema.default_max_page_size].compact.min + @limit_value ||= [first, last, max_page_size || GitlabSchema.default_max_page_size].compact.min end def loaded?(items) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 5b9216c0914..a2d06b7f5b3 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,28 +44,28 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 38, + 'da_DK' => 37, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 37, + 'es' => 36, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 72, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 31, - 'ko' => 17, - 'nb_NO' => 26, + 'ko' => 20, + 'nb_NO' => 25, 'nl_NL' => 0, - 'pl_PL' => 4, - 'pt_BR' => 56, + 'pl_PL' => 3, + 'pt_BR' => 57, 'ro_RO' => 99, - 'ru' => 27, - 'si_LK' => 10, + 'ru' => 26, + 'si_LK' => 11, 'tr_TR' => 11, - 'uk' => 50, - 'zh_CN' => 97, + 'uk' => 49, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 6c5fba37d7b..fe0ab01e4fd 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -26,10 +26,10 @@ module Gitlab log_info(message: "Started uploading project", export_size: export_size) upload_duration = Benchmark.realtime do - if Feature.enabled?(:import_export_web_upload_stream) && !project.export_file.file_storage? - upload_project_as_remote_stream - else + if project.export_file.file_storage? handle_response_error(send_file) + else + upload_project_as_remote_stream end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index bbec473d29d..b05d9cb2489 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -15,19 +15,19 @@ module Gitlab UNIQUE_RELATIONS = %i[].freeze USER_REFERENCES = %w[ - author_id - assignee_id - updated_by_id - merged_by_id - latest_closed_by_id - user_id - created_by_id - last_edited_by_id - merge_user_id - resolved_by_id - closed_by_id - owner_id - ].freeze + author_id + assignee_id + updated_by_id + merged_by_id + latest_closed_by_id + user_id + created_by_id + last_edited_by_id + merge_user_id + resolved_by_id + closed_by_id + owner_id + ].freeze TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 99396d64779..cf62f181366 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -175,21 +175,22 @@ module Gitlab order_expression = arel_table[column].public_send(direction).public_send(nulls_position) # rubocop:disable GitlabSecurity/PublicSend reverse_order_expression = arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position) # rubocop:disable GitlabSecurity/PublicSend - ::Gitlab::Pagination::Keyset::Order.build([ - ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: column, - column_expression: arel_table[column], - order_expression: order_expression, - reversed_order_expression: reverse_order_expression, - order_direction: direction, - nullable: nulls_position, - distinct: false - ), - ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: klass.primary_key, - order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) - ) - ]) + ::Gitlab::Pagination::Keyset::Order.build( + [ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: arel_table[column], + order_expression: order_expression, + reversed_order_expression: reverse_order_expression, + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: klass.primary_key, + order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) + ) + ]) end def read_from_replica_if_available(&block) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 33e4823f192..fb44aaf094e 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -98,6 +98,7 @@ tree: - :statuses - :external_pull_request - :merge_request + - :pipeline_metadata - :auto_devops - :pipeline_schedules - :container_expiration_policy @@ -582,6 +583,9 @@ included_attributes: - :iid - :source_sha - :target_sha + pipeline_metadata: + - :project_id + - :title stages: - :name - :status @@ -971,6 +975,8 @@ excluded_attributes: - :external_pull_request_id - :ci_ref_id - :locked + pipeline_metadata: + - :pipeline_id stages: - :pipeline_id merge_access_levels: diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 89f2b36ea58..4ea47a5624a 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -64,7 +64,7 @@ module Gitlab end def execute_sidekiq_job - Sidekiq::Worker.drain_all + Sidekiq::Worker.drain_all # rubocop:disable Cop/SidekiqApiUsage end def full_path diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index c4b0e24e34a..568315930d8 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -13,6 +13,7 @@ module Gitlab pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', runners: 'Ci::Runner', + pipeline_metadata: 'Ci::PipelineMetadata', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index ad19508fb99..bc0563729a7 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -86,8 +86,9 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) download_or_copy_upload(upload, upload_path) - rescue Errno::ENAMETOOLONG => e - # Do not fail entire project export if downloaded file has filename that exceeds 255 characters. + rescue StandardError => e + # Do not fail entire project export if something goes wrong during file download + # (e.g. downloaded file has filename that exceeds 255 characters). # Ignore raised exception, skip such upload, log the error and keep going with the export instead. Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) end diff --git a/lib/gitlab/jira_import/handle_labels_service.rb b/lib/gitlab/jira_import/handle_labels_service.rb index 1b00515cced..60d7f9e93d9 100644 --- a/lib/gitlab/jira_import/handle_labels_service.rb +++ b/lib/gitlab/jira_import/handle_labels_service.rb @@ -12,7 +12,7 @@ module Gitlab return if jira_labels.blank? existing_labels = LabelsFinder.new(nil, project: project, title: jira_labels) - .execute(skip_authorization: true).select(:id, :name) + .execute(skip_authorization: true).select(:id, :project_id, :group_id, :type, :name) new_labels = create_missing_labels(existing_labels) label_ids = existing_labels.map(&:id) diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 25dffcbe0ee..5057317ae01 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -7,10 +7,6 @@ module Gitlab # see https://jira.atlassian.com/browse/JRACLOUD-67570 # We set it to 1000 in case they change their mind. BATCH_SIZE = 1000 - JIRA_IMPORT_THRESHOLD = 100_000 - JIRA_IMPORT_PAUSE_LIMIT = 50_000 - - RetriesExceededError = Class.new(RuntimeError) attr_reader :imported_items_cache_key, :start_at, :job_waiter @@ -52,7 +48,7 @@ module Gitlab end def schedule_issue_import_workers(issues) - next_iid = project.issues.maximum(:iid).to_i + 1 + next_iid = Issue.with_project_iid_supply(project, &:next_value) issues.each do |jira_issue| # Technically it's possible that the same work is performed multiple @@ -71,13 +67,11 @@ module Gitlab { iid: next_iid } ).execute - # Pause the importer to allow the import to catch up and cache to drain - pause_jira_issue_importer if jira_import_issue_worker.queue_size > JIRA_IMPORT_THRESHOLD - Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 - next_iid += 1 + + next_iid = Issue.with_project_iid_supply(project, &:next_value) # Mark the issue as imported immediately so we don't end up # importing it multiple times within same import. @@ -97,27 +91,6 @@ module Gitlab job_waiter end - def jira_import_issue_worker - @_jira_import_issue_worker ||= Gitlab::JiraImport::ImportIssueWorker - end - - def pause_jira_issue_importer - # Wait for import workers to drop below 50K in the iterations of the timeout - # timeout - Set to 5 seconds. - # Time to process 100K jobs is currently ~14 seconds. - # Source: https://github.com/mperham/sidekiq#performance - # retries - Set to 10 times to avoid indefinitely pause. - # Raises an error if the queue does not reduce below the limit after 10 tries. - - retries = 10 - while retries > 0 && jira_import_issue_worker.queue_size >= JIRA_IMPORT_PAUSE_LIMIT - job_waiter.wait(5) - retries -= 1 - end - - raise RetriesExceededError, 'Retry failed after 10 attempts' if retries == 0 - end - def fetch_issues(start_at) client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at }) end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index ce07752f88c..823d6202b1e 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -34,6 +34,7 @@ module Gitlab alias_method :parse!, :parse alias_method :load, :parse + alias_method :decode, :parse # Restricted method for converting a Ruby object to JSON. If you # need to pass options to this, you should use `.generate` instead, @@ -56,6 +57,8 @@ module Gitlab adapter_generate(object, opts) end + alias_method :encode, :generate + # Generates JSON for an object and makes it look purdy # # The Oj variant in this looks seriously weird but these are the settings diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index fa10e922c80..6799be8e279 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -6,13 +6,13 @@ module Gitlab # Helper methods for Kroki module Kroki BLOCKDIAG_FORMATS = %w[ - blockdiag - seqdiag - actdiag - nwdiag - packetdiag - rackdiag - ].freeze + blockdiag + seqdiag + actdiag + nwdiag + packetdiag + rackdiag + ].freeze DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w(mermaid)).freeze DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w(plantuml)).freeze diff --git a/lib/gitlab/legacy_github_import/base_formatter.rb b/lib/gitlab/legacy_github_import/base_formatter.rb index 0b19cf742ed..7bb33cd474b 100644 --- a/lib/gitlab/legacy_github_import/base_formatter.rb +++ b/lib/gitlab/legacy_github_import/base_formatter.rb @@ -23,7 +23,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def url - raw_data.url || '' + raw_data[:url] || '' end end end diff --git a/lib/gitlab/legacy_github_import/branch_formatter.rb b/lib/gitlab/legacy_github_import/branch_formatter.rb index 1177751457f..372c6b2e8a0 100644 --- a/lib/gitlab/legacy_github_import/branch_formatter.rb +++ b/lib/gitlab/legacy_github_import/branch_formatter.rb @@ -3,7 +3,17 @@ module Gitlab module LegacyGithubImport class BranchFormatter < BaseFormatter - delegate :repo, :sha, :ref, to: :raw_data + def repo + raw_data[:repo] + end + + def sha + raw_data[:sha] + end + + def ref + raw_data[:ref] + end def exists? branch_exists? && commit_exists? @@ -14,7 +24,7 @@ module Gitlab end def user - raw_data.user&.login || 'unknown' + raw_data.dig(:user, :login) || 'unknown' end def short_sha diff --git a/lib/gitlab/legacy_github_import/comment_formatter.rb b/lib/gitlab/legacy_github_import/comment_formatter.rb index d83cc4f6b3c..ffd9da604ca 100644 --- a/lib/gitlab/legacy_github_import/comment_formatter.rb +++ b/lib/gitlab/legacy_github_import/comment_formatter.rb @@ -9,19 +9,19 @@ module Gitlab { project: project, note: note, - commit_id: raw_data.commit_id, + commit_id: raw_data[:commit_id], line_code: line_code, author_id: author_id, type: type, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end private def author - @author ||= UserFormatter.new(client, raw_data.user) + @author ||= UserFormatter.new(client, raw_data[:user]) end def author_id @@ -29,7 +29,7 @@ module Gitlab end def body - raw_data.body || "" + raw_data[:body] || "" end def line_code @@ -48,11 +48,11 @@ module Gitlab end def diff_hunk - raw_data.diff_hunk + raw_data[:diff_hunk] end def file_path - raw_data.path + raw_data[:path] end def note diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 4ddafbac4c6..331eab7b62a 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -96,7 +96,7 @@ module Gitlab def import_labels fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| - gh_label = LabelFormatter.new(project, raw) + gh_label = LabelFormatter.new(project, raw.to_h) gh_label.create! rescue StandardError => e errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } @@ -109,7 +109,7 @@ module Gitlab def import_milestones fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| - gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone = MilestoneFormatter.new(project, raw.to_h) gh_milestone.create! rescue StandardError => e errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } @@ -121,6 +121,7 @@ module Gitlab def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| + raw = raw.to_h gh_issue = IssueFormatter.new(project, raw, client) begin @@ -143,6 +144,7 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| + raw = raw.to_h gh_pull_request = PullRequestFormatter.new(project, raw, client) next unless gh_pull_request.valid? @@ -190,10 +192,12 @@ module Gitlab end def apply_labels(issuable, raw) - return unless raw.labels.count > 0 + raw = raw.to_h - label_ids = raw.labels - .map { |attrs| @labels[attrs.name] } + return unless raw[:labels].count > 0 + + label_ids = raw[:labels] + .map { |attrs| @labels[attrs[:name]] } .compact issuable.update_attribute(:label_ids, label_ids) @@ -226,10 +230,12 @@ module Gitlab def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| + raw = raw.to_h + comment = CommentFormatter.new(project, raw, client) # GH does not return info about comment's parent, so we guess it by checking its URL! - *_, parent, iid = URI(raw.html_url).path.split('/') + *_, parent, iid = URI(raw[:html_url]).path.split('/') issuable = if parent == 'issues' Issue.find_by(project_id: project.id, iid: iid) @@ -241,7 +247,7 @@ module Gitlab issuable.notes.create!(comment.attributes) rescue StandardError => e - errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw[:url]), errors: e.message } end end end @@ -251,7 +257,7 @@ module Gitlab last_note_attrs = nil cut_off_index = comments.find_index do |raw| - comment = CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw.to_h) comment_attrs = comment.attributes last_note_attrs ||= last_note.slice(*comment_attrs.keys) @@ -282,7 +288,7 @@ module Gitlab def import_releases fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| - gh_release = ReleaseFormatter.new(project, raw) + gh_release = ReleaseFormatter.new(project, raw.to_h) gh_release.create! if gh_release.valid? rescue StandardError => e errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } diff --git a/lib/gitlab/legacy_github_import/issuable_formatter.rb b/lib/gitlab/legacy_github_import/issuable_formatter.rb index 1a0aefbbd62..e4e333735be 100644 --- a/lib/gitlab/legacy_github_import/issuable_formatter.rb +++ b/lib/gitlab/legacy_github_import/issuable_formatter.rb @@ -9,7 +9,9 @@ module Gitlab raise NotImplementedError end - delegate :number, to: :raw_data + def number + raw_data[:number] + end def find_condition { iid: number } @@ -18,15 +20,15 @@ module Gitlab private def state - raw_data.state == 'closed' ? 'closed' : 'opened' + raw_data[:state] == 'closed' ? 'closed' : 'opened' end def assigned? - raw_data.assignee.present? + raw_data[:assignee].present? end def author - @author ||= UserFormatter.new(client, raw_data.user) + @author ||= UserFormatter.new(client, raw_data[:user]) end def author_id @@ -35,7 +37,7 @@ module Gitlab def assignee if assigned? - @assignee ||= UserFormatter.new(client, raw_data.assignee) + @assignee ||= UserFormatter.new(client, raw_data[:assignee]) end end @@ -46,7 +48,7 @@ module Gitlab end def body - raw_data.body || "" + raw_data[:body] || "" end def description @@ -59,8 +61,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def milestone - if raw_data.milestone.present? - milestone = MilestoneFormatter.new(project, raw_data.milestone) + if raw_data[:milestone].present? + milestone = MilestoneFormatter.new(project, raw_data[:milestone]) project.milestones.find_by(milestone.find_condition) end end diff --git a/lib/gitlab/legacy_github_import/issue_formatter.rb b/lib/gitlab/legacy_github_import/issue_formatter.rb index 2f46e2e30d1..e5c568207e3 100644 --- a/lib/gitlab/legacy_github_import/issue_formatter.rb +++ b/lib/gitlab/legacy_github_import/issue_formatter.rb @@ -8,18 +8,18 @@ module Gitlab iid: number, project: project, milestone: milestone, - title: raw_data.title, + title: raw_data[:title], description: description, state: state, author_id: author_id, assignee_ids: Array(assignee_id), - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end def has_comments? - raw_data.comments > 0 + raw_data[:comments] > 0 end def project_association @@ -27,7 +27,7 @@ module Gitlab end def pull_request? - raw_data.pull_request.present? + raw_data[:pull_request].present? end end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 415b1b8878f..e3b767f41fa 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -28,11 +28,11 @@ module Gitlab private def color - "##{raw_data.color}" + "##{raw_data[:color]}" end def title - raw_data.name + raw_data[:name] end end end diff --git a/lib/gitlab/legacy_github_import/milestone_formatter.rb b/lib/gitlab/legacy_github_import/milestone_formatter.rb index 2fe1b4258d3..60d5bcbf44a 100644 --- a/lib/gitlab/legacy_github_import/milestone_formatter.rb +++ b/lib/gitlab/legacy_github_import/milestone_formatter.rb @@ -7,12 +7,12 @@ module Gitlab { iid: number, project: project, - title: raw_data.title, - description: raw_data.description, - due_date: raw_data.due_on, + title: raw_data[:title], + description: raw_data[:description], + due_date: raw_data[:due_on], state: state, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at] } end @@ -26,16 +26,16 @@ module Gitlab def number if project.gitea_import? - raw_data.id + raw_data[:id] else - raw_data.number + raw_data[:number] end end private def state - raw_data.state == 'closed' ? 'closed' : 'active' + raw_data[:state] == 'closed' ? 'closed' : 'active' end end end diff --git a/lib/gitlab/legacy_github_import/pull_request_formatter.rb b/lib/gitlab/legacy_github_import/pull_request_formatter.rb index 5b847f13d4a..72adba30093 100644 --- a/lib/gitlab/legacy_github_import/pull_request_formatter.rb +++ b/lib/gitlab/legacy_github_import/pull_request_formatter.rb @@ -9,7 +9,7 @@ module Gitlab def attributes { iid: number, - title: raw_data.title, + title: raw_data[:title], description: description, source_project: source_branch_project, source_branch: source_branch_name, @@ -21,8 +21,8 @@ module Gitlab milestone: milestone, author_id: author_id, assignee_id: assignee_id, - created_at: raw_data.created_at, - updated_at: raw_data.updated_at, + created_at: raw_data[:created_at], + updated_at: raw_data[:updated_at], imported: true } end @@ -36,7 +36,7 @@ module Gitlab end def source_branch - @source_branch ||= BranchFormatter.new(project, raw_data.head) + @source_branch ||= BranchFormatter.new(project, raw_data[:head]) end def source_branch_name @@ -57,7 +57,7 @@ module Gitlab end def target_branch - @target_branch ||= BranchFormatter.new(project, raw_data.base) + @target_branch ||= BranchFormatter.new(project, raw_data[:base]) end def target_branch_name @@ -71,7 +71,7 @@ module Gitlab def cross_project? return true if source_branch_repo.nil? - source_branch_repo.id != target_branch_repo.id + source_branch_repo[:id] != target_branch_repo[:id] end def opened? @@ -81,7 +81,7 @@ module Gitlab private def state - if raw_data.state == 'closed' && raw_data.merged_at.present? + if raw_data[:state] == 'closed' && raw_data[:merged_at].present? 'merged' else super diff --git a/lib/gitlab/legacy_github_import/release_formatter.rb b/lib/gitlab/legacy_github_import/release_formatter.rb index 0fb7e376f5b..2a54a15429b 100644 --- a/lib/gitlab/legacy_github_import/release_formatter.rb +++ b/lib/gitlab/legacy_github_import/release_formatter.rb @@ -6,13 +6,13 @@ module Gitlab def attributes { project: project, - tag: raw_data.tag_name, - name: raw_data.name, - description: raw_data.body, - created_at: raw_data.created_at, + tag: raw_data[:tag_name], + name: raw_data[:name], + description: raw_data[:body], + created_at: raw_data[:created_at], # Draft releases will have a null published_at - released_at: raw_data.published_at || Time.current, - updated_at: raw_data.created_at + released_at: raw_data[:published_at] || Time.current, + updated_at: raw_data[:created_at] } end @@ -21,11 +21,11 @@ module Gitlab end def find_condition - { tag: raw_data.tag_name } + { tag: raw_data[:tag_name] } end def valid? - !raw_data.draft && raw_data.tag_name.present? + !raw_data[:draft] && raw_data[:tag_name].present? end end end diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index 7ae1b195ec6..d45a166d2b7 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -5,13 +5,19 @@ module Gitlab class UserFormatter attr_reader :client, :raw - delegate :id, :login, to: :raw, allow_nil: true - def initialize(client, raw) @client = client @raw = raw end + def id + raw[:id] + end + + def login + raw[:login] + end + def gitlab_id return @gitlab_id if defined?(@gitlab_id) @@ -21,7 +27,7 @@ module Gitlab private def email - @email ||= client.user(raw.login).try(:email) + @email ||= client.user(raw[:login]).to_h[:email] end def find_by_email diff --git a/lib/gitlab/memory/diagnostic_reports_logger.rb b/lib/gitlab/memory/diagnostic_reports_logger.rb new file mode 100644 index 00000000000..cc5b719fa19 --- /dev/null +++ b/lib/gitlab/memory/diagnostic_reports_logger.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'logger' + +module Gitlab + module Memory + class DiagnosticReportsLogger < ::Logger + def format_message(severity, timestamp, progname, message) + data = {} + data[:severity] = severity + data[:time] = timestamp.utc.iso8601(3) + + data.merge!(message) + + "#{JSON.generate(data)}\n" # rubocop:disable Gitlab/Json + end + end + end +end diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index ed1da8baab5..0dfc31235e7 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_SLEEP_MAX_DELTA_S = 600 # 0..10 minutes DEFAULT_SLEEP_BETWEEN_REPORTS_S = 120 # 2 minutes - DEFAULT_REPORTS_PATH = '/tmp' + DEFAULT_REPORTS_PATH = Dir.tmpdir def initialize(**options) super diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb new file mode 100644 index 00000000000..76c3e0862e2 --- /dev/null +++ b/lib/gitlab/memory/reports_uploader.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require_relative '../metrics/system' + +module Gitlab + module Memory + class ReportsUploader + def initialize(gcs_key:, gcs_project:, gcs_bucket:, logger:) + @gcs_bucket = gcs_bucket + @fog = Fog::Storage::Google.new(google_project: gcs_project, google_json_key_location: gcs_key) + @logger = logger + end + + def upload(path) + log_upload_requested(path) + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + + File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) } + + duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + log_upload_success(path, duration_s) + rescue StandardError, Errno::ENOENT => error + log_exception(error) + end + + private + + attr_reader :gcs_bucket, :fog, :logger + + def log_upload_requested(path) + logger.info(log_labels.merge(perf_report_status: 'upload requested', perf_report_path: path)) + end + + def log_upload_success(path, duration_s) + logger.info(log_labels.merge(perf_report_status: 'upload success', perf_report_path: path, + duration_s: duration_s)) + end + + def log_exception(error) + logger.error(log_labels.merge(perf_report_status: "error", error: error.message)) + end + + def log_labels + { + message: "Diagnostic reports", + class: self.class.name, + pid: $$ + } + end + end + end +end diff --git a/lib/gitlab/memory/upload_and_cleanup_reports.rb b/lib/gitlab/memory/upload_and_cleanup_reports.rb new file mode 100644 index 00000000000..27d94df478c --- /dev/null +++ b/lib/gitlab/memory/upload_and_cleanup_reports.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class UploadAndCleanupReports + DEFAULT_SLEEP_TIME_SECONDS = 900 # 15 minutes + + def initialize( + uploader:, + reports_path:, + logger:, + sleep_time_seconds: ENV['GITLAB_DIAGNOSTIC_REPORTS_UPLOADER_SLEEP_S']&.to_i || DEFAULT_SLEEP_TIME_SECONDS) + + @uploader = uploader + @reports_path = reports_path + @sleep_time_seconds = sleep_time_seconds + @alive = true + @logger = logger + end + + attr_reader :uploader, :reports_path, :sleep_time_seconds, :logger + + def call + log_started + + loop do + sleep(sleep_time_seconds) + + files_to_process.each { |path| upload_and_cleanup!(path) } + end + end + + private + + def upload_and_cleanup!(path) + uploader.upload(path) + rescue StandardError, Errno::ENOENT => error + log_exception(error) + ensure + cleanup!(path) + end + + def cleanup!(path) + File.unlink(path) if File.exist?(path) + rescue Errno::ENOENT + # Path does not exist: Ignore. We already check `File.exist?`. Rescue to be extra safe. + end + + def files_to_process + Dir.entries(reports_path) + .map { |path| File.join(reports_path, path) } + .select { |path| File.file?(path) } + end + + def log_started + logger.info(log_labels.merge(perf_report_status: "started")) + end + + def log_exception(error) + logger.error(log_labels.merge(perf_report_status: "error", error: error.message)) + end + + def log_labels + { + message: "Diagnostic reports", + class: self.class.name, + pid: $$ + } + end + end + end +end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 38231fa933b..7007fdfe386 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -2,25 +2,10 @@ module Gitlab module Memory - # A background thread that observes Ruby heap fragmentation and calls - # into a handler when the Ruby heap has been fragmented for an extended - # period of time. - # - # See Gitlab::Metrics::Memory for how heap fragmentation is defined. - # - # To decide whether a given fragmentation level is being exceeded, - # the watchdog regularly polls the GC. Whenever a violation occurs - # a strike is issued. If the maximum number of strikes are reached, - # a handler is invoked to deal with the situation. - # - # The duration for which a process may be above a given fragmentation - # threshold is computed as `max_strikes * sleep_time_seconds`. + # A background thread that monitors Ruby memory and calls + # into a handler when the Ruby process violates defined limits + # for an extended period of time. class Watchdog - DEFAULT_SLEEP_TIME_SECONDS = 60 * 5 - DEFAULT_MAX_HEAP_FRAG = 0.5 - DEFAULT_MAX_MEM_GROWTH = 3.0 - DEFAULT_MAX_STRIKES = 5 - # This handler does nothing. It returns `false` to indicate to the # caller that the situation has not been dealt with so it will # receive calls repeatedly if fragmentation remains high. @@ -62,73 +47,27 @@ module Gitlab end end - # max_heap_fragmentation: - # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. - # max_mem_growth: - # A multiplier for how much excess private memory a worker can map compared to a reference process - # (itself or the primary in a pre-fork server.) - # max_strikes: - # How many times the process is allowed to be above max_heap_fragmentation before - # a handler is invoked. - # sleep_time_seconds: - # Used to control the frequency with which the watchdog will wake up and poll the GC. - def initialize( - handler: NullHandler.instance, - logger: Logger.new($stdout), - max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_MAX_HEAP_FRAG, - max_mem_growth: ENV['GITLAB_MEMWD_MAX_MEM_GROWTH']&.to_f || DEFAULT_MAX_MEM_GROWTH, - max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, - sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, - **options) - super(**options) - - @handler = handler - @logger = logger - @sleep_time_seconds = sleep_time_seconds - @max_strikes = max_strikes - @stats = { - heap_frag: { - max: max_heap_fragmentation, - strikes: 0 - }, - mem_growth: { - max: max_mem_growth, - strikes: 0 - } - } - + def initialize + @configuration = Configuration.new @alive = true - init_prometheus_metrics(max_heap_fragmentation) - end - - attr_reader :max_strikes, :sleep_time_seconds - - def max_heap_fragmentation - @stats[:heap_frag][:max] - end - - def max_mem_growth - @stats[:mem_growth][:max] + init_prometheus_metrics end - def strikes(stat) - @stats[stat][:strikes] + def configure + yield @configuration end def call - @logger.info(log_labels.merge(message: 'started')) + logger.info(log_labels.merge(message: 'started')) while @alive - sleep(@sleep_time_seconds) - - next unless Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + sleep(sleep_time_seconds) - monitor_heap_fragmentation - monitor_memory_growth + monitor if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) end - @logger.info(log_labels.merge(message: 'stopped')) + logger.info(log_labels.merge(message: 'stopped')) end def stop @@ -137,71 +76,24 @@ module Gitlab private - def monitor_memory_condition(stat_key) - return unless @alive - - stat = @stats[stat_key] - - ok, labels = yield(stat) + def monitor + @configuration.monitors.call_each do |result| + break unless @alive - if ok - stat[:strikes] = 0 - else - stat[:strikes] += 1 - @counter_violations.increment(reason: stat_key.to_s) - end + next unless result.threshold_violated? - if stat[:strikes] > @max_strikes - @alive = !memory_limit_exceeded_callback(stat_key, labels) - stat[:strikes] = 0 - end - end + @counter_violations.increment(reason: result.monitor_name) - def monitor_heap_fragmentation - monitor_memory_condition(:heap_frag) do |stat| - heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation - [ - heap_fragmentation <= stat[:max], - { - message: 'heap fragmentation limit exceeded', - memwd_cur_heap_frag: heap_fragmentation, - memwd_max_heap_frag: stat[:max] - } - ] - end - end + next unless result.strikes_exceeded? - def monitor_memory_growth - monitor_memory_condition(:mem_growth) do |stat| - worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] - reference_uss = reference_mem[:uss] - memory_limit = stat[:max] * reference_uss - [ - worker_uss <= memory_limit, - { - message: 'memory limit exceeded', - memwd_uss_bytes: worker_uss, - memwd_ref_uss_bytes: reference_uss, - memwd_max_uss_bytes: memory_limit - } - ] + @alive = !memory_limit_exceeded_callback(result.monitor_name, result.payload) end end - # On pre-fork systems this would be the primary process memory from which workers fork. - # Otherwise it is the current process' memory. - # - # We initialize this lazily because in the initializer the application may not have - # finished booting yet, which would yield an incorrect baseline. - def reference_mem - @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) - end - - def memory_limit_exceeded_callback(stat_key, handler_labels) - all_labels = log_labels.merge(handler_labels) - .merge(memwd_cur_strikes: strikes(stat_key)) - @logger.warn(all_labels) - @counter_violations_handled.increment(reason: stat_key.to_s) + def memory_limit_exceeded_callback(monitor_name, monitor_payload) + all_labels = log_labels.merge(monitor_payload) + logger.warn(all_labels) + @counter_violations_handled.increment(reason: monitor_name) handler.call end @@ -211,7 +103,15 @@ module Gitlab # all that happens is we collect logs and Prometheus events for fragmentation violations. return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - @handler + @configuration.handler + end + + def logger + @configuration.logger + end + + def sleep_time_seconds + @configuration.sleep_time_seconds end def log_labels @@ -219,27 +119,20 @@ module Gitlab pid: $$, worker_id: worker_id, memwd_handler_class: handler.class.name, - memwd_sleep_time_s: @sleep_time_seconds, - memwd_max_strikes: @max_strikes, + memwd_sleep_time_s: sleep_time_seconds, memwd_rss_bytes: process_rss_bytes } end - def worker_id - ::Prometheus::PidProvider.worker_id - end - def process_rss_bytes Gitlab::Metrics::System.memory_usage_rss end - def init_prometheus_metrics(max_heap_fragmentation) - @heap_frag_limit = Gitlab::Metrics.gauge( - :gitlab_memwd_heap_frag_limit, - 'The configured limit for how fragmented the Ruby heap is allowed to be' - ) - @heap_frag_limit.set({}, max_heap_fragmentation) + def worker_id + ::Prometheus::PidProvider.worker_id + end + def init_prometheus_metrics default_labels = { pid: worker_id } @counter_violations = Gitlab::Metrics.counter( :gitlab_memwd_violations_total, diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb new file mode 100644 index 00000000000..2d84b083f55 --- /dev/null +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class Configuration + class MonitorStack + def initialize + @monitors = [] + end + + def use(monitor_class, *args, **kwargs, &block) + remove(monitor_class) + @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) + end + + def call_each + @monitors.each do |monitor| + yield monitor.call + end + end + + private + + def remove(monitor_class) + @monitors.delete_if { |monitor| monitor.monitor_class == monitor_class } + end + + def build_monitor_state(monitor_class, *args, max_strikes:, **kwargs, &block) + monitor = build_monitor(monitor_class, *args, **kwargs, &block) + + Gitlab::Memory::Watchdog::MonitorState.new(monitor, max_strikes: max_strikes) + end + + def build_monitor(monitor_class, *args, **kwargs, &block) + monitor_class.new(*args, **kwargs, &block) + end + end + + DEFAULT_SLEEP_TIME_SECONDS = 60 + + attr_reader :monitors + attr_writer :logger, :handler, :sleep_time_seconds + + def initialize + @monitors = MonitorStack.new + end + + def handler + @handler ||= NullHandler.instance + end + + def logger + @logger ||= Gitlab::Logger.new($stdout) + end + + # Used to control the frequency with which the watchdog will wake up and poll the GC. + def sleep_time_seconds + @sleep_time_seconds ||= DEFAULT_SLEEP_TIME_SECONDS + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb new file mode 100644 index 00000000000..7748c19c6d8 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + # A monitor that observes Ruby heap fragmentation and calls + # memory_violation_callback when the Ruby heap has been fragmented for an extended + # period of time. + # + # See Gitlab::Metrics::Memory for how heap fragmentation is defined. + class HeapFragmentation + attr_reader :max_heap_fragmentation + + # max_heap_fragmentation: + # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + def initialize(max_heap_fragmentation:) + @max_heap_fragmentation = max_heap_fragmentation + init_frag_limit_metrics + end + + def call + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + + return { threshold_violated: false, payload: {} } unless heap_fragmentation > max_heap_fragmentation + + { threshold_violated: true, payload: payload(heap_fragmentation) } + end + + private + + def payload(heap_fragmentation) + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: heap_fragmentation, + memwd_max_heap_frag: max_heap_fragmentation + } + end + + def init_frag_limit_metrics + heap_frag_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_heap_frag_limit, + 'The configured limit for how fragmented the Ruby heap is allowed to be' + ) + heap_frag_limit.set({}, max_heap_fragmentation) + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb new file mode 100644 index 00000000000..2a1512c4cff --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + class UniqueMemoryGrowth + attr_reader :max_mem_growth + + def initialize(max_mem_growth:) + @max_mem_growth = max_mem_growth + end + + def call + worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] + reference_uss = reference_mem[:uss] + memory_limit = max_mem_growth * reference_uss + + return { threshold_violated: false, payload: {} } unless worker_uss > memory_limit + + { threshold_violated: true, payload: payload(worker_uss, reference_uss, memory_limit) } + end + + private + + def payload(worker_uss, reference_uss, memory_limit) + { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_uss, + memwd_ref_uss_bytes: reference_uss, + memwd_max_uss_bytes: memory_limit + } + end + + # On pre-fork systems this would be the primary process memory from which workers fork. + # Otherwise it is the current process' memory. + # + # We initialize this lazily because in the initializer the application may not have + # finished booting yet, which would yield an incorrect baseline. + def reference_mem + @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor_state.rb b/lib/gitlab/memory/watchdog/monitor_state.rb new file mode 100644 index 00000000000..73be5de3e45 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor_state.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class MonitorState + class Result + attr_reader :payload + + def initialize(strikes_exceeded:, threshold_violated:, monitor_class:, payload: ) + @strikes_exceeded = strikes_exceeded + @threshold_violated = threshold_violated + @monitor_class = monitor_class + @payload = payload + end + + def strikes_exceeded? + @strikes_exceeded + end + + def threshold_violated? + @threshold_violated + end + + def monitor_name + @monitor_class.name.demodulize.underscore.to_sym + end + end + + def initialize(monitor, max_strikes:) + @monitor = monitor + @max_strikes = max_strikes + @strikes = 0 + end + + def call + reset_strikes if strikes_exceeded? + + monitor_result = @monitor.call + + if monitor_result[:threshold_violated] + issue_strike + else + reset_strikes + end + + build_result(monitor_result) + end + + def monitor_class + @monitor.class + end + + private + + def build_result(monitor_result) + Result.new( + strikes_exceeded: strikes_exceeded?, + monitor_class: monitor_class, + threshold_violated: monitor_result[:threshold_violated], + payload: payload.merge(monitor_result[:payload])) + end + + def payload + { + memwd_max_strikes: @max_strikes, + memwd_cur_strikes: @strikes + } + end + + def strikes_exceeded? + @strikes > @max_strikes + end + + def issue_strike + @strikes += 1 + end + + def reset_strikes + @strikes = 0 + end + end + end + end +end diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index e37129fed38..3400a6c78ef 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -13,9 +13,7 @@ module Gitlab ADVANCED_CODE_TARGET_S = 13.546 def initialize_slis! - if Feature.enabled?(:global_search_custom_slis) - Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) - end + Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) return unless Feature.enabled?(:global_search_error_rate_sli) @@ -23,8 +21,6 @@ module Gitlab end def record_apdex(elapsed:, search_type:, search_level:, search_scope:) - return unless Feature.enabled?(:global_search_custom_slis) - Gitlab::Metrics::Sli::Apdex[:global_search].increment( labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), success: elapsed < duration_target(search_type, search_scope) diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index d7eef722d6e..affadc4274c 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -14,11 +14,13 @@ module Gitlab PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' + PROC_MEM_INFO = '/proc/meminfo' PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze + MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze def summary proportional_mem = memory_usage_uss_pss @@ -45,6 +47,10 @@ module Gitlab .transform_values(&:kilobytes) end + def memory_total + sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes + end + def file_descriptor_count Dir.glob(PROC_FD_GLOB).length end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 0576aed811c..12d4d9d8928 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -138,9 +138,9 @@ module Gitlab .with .recursive(ancestors.to_arel, descendants.to_arel) .from_union([ - ancestors_scope, - descendants_scope - ]) + ancestors_scope, + descendants_scope + ]) read_only(relation) end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index 991a1297d03..187d5f907e4 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -3,9 +3,13 @@ module Gitlab module Pages class CacheControl - CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}' + include Gitlab::Utils::StrongMemoize - attr_reader :cache_key + EXPIRE = 12.hours + # To avoid delivering expired deployment URL in the cached payload, + # use a longer expiration time in the deployment URL + DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours) + CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}_%{settings}' class << self def for_project(project_id) @@ -20,12 +24,35 @@ module Gitlab def initialize(type:, id:) raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) - @cache_key = CACHE_KEY_FORMAT % { type: type, id: id } + @type = type + @id = id + end + + def cache_key + strong_memoize(:cache_key) do + CACHE_KEY_FORMAT % { + type: @type, + id: @id, + settings: settings + } + end end def clear_cache Rails.cache.delete(cache_key) end + + private + + def settings + values = ::Gitlab.config.pages.dup + + values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( + 'force_pages_access_control' + ) + + ::Digest::SHA256.hexdigest(values.inspect) + end end end end diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index c36bd497aa3..318720c77d1 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -129,28 +129,31 @@ module Gitlab end def primary_key_descending_order - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: model_class.primary_key, - order_expression: arel_table[primary_key].desc - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + ]) end def primary_key_order - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: model_class.primary_key, - order_expression: order_values.first - ) - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: order_values.first + ) + ]) end def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) - Gitlab::Pagination::Keyset::Order.build([ - column(order_values.first), - tie_breaker_column_order - ]) + Gitlab::Pagination::Keyset::Order.build( + [ + column(order_values.first), + tie_breaker_column_order + ]) end def column(order_value) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb deleted file mode 100644 index 630c364d455..00000000000 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -# Patch to address https://github.com/ondrejbartas/sidekiq-cron/issues/361 -# This restores the poll interval to v1.2.0 behavior -# https://github.com/ondrejbartas/sidekiq-cron/blob/v1.2.0/lib/sidekiq/cron/poller.rb#L36-L38 -# This patch only applies to v1.4.0 -require 'sidekiq/cron/version' - -if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.4.0') - raise 'New version of sidekiq-cron detected, please remove or update this patch' -end - -module Gitlab - module Patch - module SidekiqCronPoller - def poll_interval_average - Sidekiq.options[:poll_interval] || Sidekiq::Cron::POLL_INTERVAL - end - end - end -end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index fd9f73d18c1..f8a85f693bc 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -43,12 +43,9 @@ module Gitlab # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. # - # - sampling_mode: When true, uses a sampling profiler (StackProf) instead of a tracing profiler (RubyProf). - # - # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults by profiler type: - # RubyProf - {} - # StackProf - { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } - def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, sampling_mode: false, profiler_options: {}) + # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults: + # { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, profiler_options: {}) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get headers = {} @@ -80,7 +77,7 @@ module Gitlab with_custom_logger(logger) do with_user(user) do - with_profiler(sampling_mode, profiler_options) do + with_profiler(profiler_options) do app.public_send(verb, url, params: post_data, headers: headers) # rubocop:disable GitlabSecurity/PublicSend end end @@ -174,21 +171,11 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def self.print_by_total_time(result, options = {}) - default_options = { sort_method: :total_time, filter_by: :total_time } - - RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) - end - - def self.with_profiler(sampling_mode, profiler_options) - if sampling_mode - require 'stackprof' - args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) - args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path - ::StackProf.run(**args) { yield } - else - RubyProf.profile(**profiler_options) { yield } - end + def self.with_profiler(profiler_options) + require 'stackprof' + args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) + args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path + ::StackProf.run(**args) { yield } end end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index 1d7b179baf0..da3f67dde51 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -37,9 +37,9 @@ module Gitlab Namespace .unscoped .select([ - links[:project_id], - least(cte_alias[:access_level], links[:group_access], 'access_level') - ]) + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) .from(cte_alias) .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') @@ -79,9 +79,9 @@ module Gitlab # Sub groups of any groups the user is a member of. cte << Group.select([ - namespaces[:id], - greatest(members[:access_level], cte.table[:access_level], 'access_level') - ]) + namespaces[:id], + greatest(members[:access_level], cte.table[:access_level], 'access_level') + ]) .joins(join_cte(cte)) .joins(join_members_on_namespaces) .except(:order) diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 2e31849caaa..46c0a0ddf7a 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -14,8 +14,13 @@ module Gitlab # The maximum number of SQL queries that can be executed in a request. For # the sake of keeping things simple we hardcode this value here, it's not # supposed to be changed very often anyway. - THRESHOLD = 100 - LOG_THRESHOLD = THRESHOLD * 1.5 + def self.threshold + 100 + end + + def self.log_threshold + threshold * 1.5 + end # Error that is raised whenever exceeding the maximum number of queries. ThresholdExceededError = Class.new(StandardError) @@ -76,7 +81,7 @@ module Gitlab end def executed_sql(sql) - return if @count > LOG_THRESHOLD || ignorable?(sql) + return if @count > self.class.log_threshold || ignorable?(sql) @sql_executed << sql end @@ -86,15 +91,15 @@ module Gitlab end def threshold_exceeded? - count > THRESHOLD + count > self.class.threshold end def error_message header = 'Too many SQL queries were executed' header = "#{header} in #{action}" if action - msg = "a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + msg = "a maximum of #{self.class.threshold} is allowed but #{count} SQL queries were executed" log = @sql_executed.each_with_index.map { |sql, i| "#{i}: #{sql}" }.join("\n").presence - ellipsis = '...' if @count > LOG_THRESHOLD + ellipsis = '...' if @count > self.class.log_threshold ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end @@ -105,3 +110,5 @@ module Gitlab end end end + +Gitlab::QueryLimiting::Transaction.prepend_mod diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index d38b81bff0b..f0ad6653c5e 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -176,7 +176,7 @@ module Gitlab explanation { _('Approve the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_approval_by?(current_user) end command :approve do success = ::MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) @@ -190,7 +190,7 @@ module Gitlab explanation { _('Unapprove the current merge request.') } types MergeRequest condition do - quick_action_target.persisted? && quick_action_target.can_be_unapproved_by?(current_user) + quick_action_target.persisted? && quick_action_target.eligible_for_unapproval_by?(current_user) end command :unapprove do success = ::MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb index beb3ba1abee..c76d298da18 100644 --- a/lib/gitlab/redis/duplicate_jobs.rb +++ b/lib/gitlab/redis/duplicate_jobs.rb @@ -18,9 +18,11 @@ module Gitlab # `Sidekiq.redis` is a namespaced redis connection. This means keys are actually being stored under # "resque:gitlab:resque:gitlab:duplicate:". For backwards compatibility, we make the secondary store # namespaced in the same way, but omit it from the primary so keys have proper format there. + # rubocop:disable Cop/RedisQueueUsage secondary_store = ::Redis::Namespace.new( Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE, redis: ::Redis.new(Gitlab::Redis::Queues.params) ) + # rubocop:enable Cop/RedisQueueUsage MultiStore.new(primary_store, secondary_store, name.demodulize) end diff --git a/lib/gitlab/redis/sidekiq_status.rb b/lib/gitlab/redis/sidekiq_status.rb index d4362c7cad8..9b8bbf5a0ad 100644 --- a/lib/gitlab/redis/sidekiq_status.rb +++ b/lib/gitlab/redis/sidekiq_status.rb @@ -14,7 +14,7 @@ module Gitlab def redis primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) + secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) # rubocop:disable Cop/RedisQueueUsage MultiStore.new(primary_store, secondary_store, name.demodulize) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 10c03103899..4f76cce2c7d 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -414,8 +414,10 @@ module Gitlab # Based on Jira's project key format # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html + # Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues. + # CVE IDs use the format of CVE-YYYY-NNNNNNN def jira_issue_key_regex - @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/ + @jira_issue_key_regex ||= /(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ end def jira_issue_key_project_key_extraction_regex diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 33c7d96c45b..baf48fd0dc1 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -41,7 +41,7 @@ module Gitlab smembers, exists = with do |redis| redis.multi do |multi| multi.smembers(full_key) - multi.exists(full_key) + multi.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -58,7 +58,7 @@ module Gitlab full_key = cache_key(key) with do |redis| - exists = redis.exists(full_key) + exists = redis.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord write(key, yield) unless exists redis.sscan_each(full_key, match: pattern) diff --git a/lib/gitlab/request_endpoints.rb b/lib/gitlab/request_endpoints.rb index 157c0f91e65..4efafaa0ac2 100644 --- a/lib/gitlab/request_endpoints.rb +++ b/lib/gitlab/request_endpoints.rb @@ -8,6 +8,7 @@ module Gitlab # but if they weren't, the routes will be drawn and available for the rest of # application. API::API.compile! + API::API.reset_routes! API::API.routes.select { |route| route.app.options[:for] < API::Base } end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 5b1341207fd..6d95cb9a87b 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -25,9 +25,9 @@ module Gitlab if matches.one? matches.first elsif matches.none? - raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$0})" + raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$PROGRAM_NAME})" else - raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" + raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$PROGRAM_NAME})" end end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 4c5fae87420..360bbc073c5 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -40,19 +40,24 @@ module Gitlab query_tokens = parse_raw_query filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = query_tokens.find { |part| part =~ /\A-?#{filter[:name]}:/ } + matches = query_tokens.select { |part| part =~ /\A-?#{filter[:name]}:/ } - next unless match + next unless matches.any? - input = match.split(':')[1..].join - next if input.empty? + matches.each do |match| + query_filter = filter.dup - filter[:negated] = match.start_with?("-") - filter[:value] = parse_filter(filter, input.gsub(QUOTES_REGEXP, '')) - filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') - fragments << match + input = match.split(':')[1..].join - parsed_filters << filter + next if input.empty? + + query_filter[:negated] = match.start_with?("-") + query_filter[:value] = parse_filter(query_filter, input.gsub(QUOTES_REGEXP, '')) + query_filter[:regex_value] = Regexp.escape(query_filter[:value]).gsub('\*', '.*?') + + fragments << match + parsed_filters << query_filter + end end query = (query_tokens - fragments).join(' ') diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 23c23393bc8..c7818cb3418 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -28,7 +28,7 @@ module Gitlab end def exist?(key) - with { |redis| redis.exists(cache_key(key)) } + with { |redis| redis.exists?(cache_key(key)) } # rubocop:disable CodeReuse/ActiveRecord end def write(key, value) @@ -59,7 +59,7 @@ module Gitlab with do |redis| redis.multi do |multi| multi.sismember(full_key, value) - multi.exists(full_key) + multi.exists?(full_key) # rubocop:disable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index ac9a7d25fc2..3e7bdfbe89a 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -53,8 +53,33 @@ module Gitlab end end + def cron_jobs + @cron_jobs ||= begin + Gitlab.config.load_dynamic_cron_schedules! + + # Load recurring jobs from gitlab.yml + # UGLY Hack to get nested hash from settingslogic + jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json) + + jobs.delete('poll_interval') # Would be interpreted as a job otherwise + + # UGLY hack: Settingslogic doesn't allow 'class' key + required_keys = %w[job_class cron] + jobs.each do |k, v| + if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) } + jobs[k]['class'] = jobs[k].delete('job_class') + else + jobs.delete(k) + Gitlab::AppLogger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") + end + end + + jobs + end + end + def cron_workers - @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize } + @cron_workers ||= cron_jobs.map { |job_name, options| options['class'].constantize } end def workers diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 24e2eca420e..b8f86b92844 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -51,9 +51,10 @@ module Gitlab def refresh_state(phase) @phase = PHASE.fetch(phase) - @current_rss = get_rss - @soft_limit_rss = get_soft_limit_rss - @hard_limit_rss = get_hard_limit_rss + @current_rss = get_rss_kb + @soft_limit_rss = get_soft_limit_rss_kb + @hard_limit_rss = get_hard_limit_rss_kb + @memory_total = get_memory_total_kb # track the current state as prometheus gauges @metrics[:sidekiq_memory_killer_phase].set({}, @phase) @@ -107,6 +108,8 @@ module Gitlab end def restart_sidekiq + return if Feature.enabled?(:sidekiq_memory_killer_read_only_mode, type: :ops) + # Tell Sidekiq to stop fetching new jobs # We first SIGNAL and then wait given time # We also monitor a number of running jobs and allow to restart early @@ -176,6 +179,7 @@ module Gitlab current_rss: @current_rss, soft_limit_rss: @soft_limit_rss, hard_limit_rss: @hard_limit_rss, + memory_total_kb: @memory_total, reason: reason, running_jobs: running_jobs) @@ -212,18 +216,19 @@ module Gitlab end end - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status&.zero? + def get_memory_total_kb + Gitlab::Metrics::System.memory_total / 1.kilobytes + end - output.to_i + def get_rss_kb + Gitlab::Metrics::System.memory_usage_rss / 1.kilobytes end - def get_soft_limit_rss + def get_soft_limit_rss_kb SOFT_LIMIT_RSS_KB + rss_increase_by_jobs end - def get_hard_limit_rss + def get_hard_limit_rss_kb HARD_LIMIT_RSS_KB end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 1f1d63877b5..655e95c82d3 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -182,7 +182,7 @@ module Gitlab def cancelled?(jid) ::Gitlab::Redis::SharedState.with do |redis| - redis.exists(self.class.cancel_job_key(jid)) + redis.exists?(self.class.cancel_job_key(jid)) # rubocop:disable CodeReuse/ActiveRecord end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index fd3a5f715e8..b20f639ce85 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true) + def self.server_configurator(metrics: true, arguments_logger: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -27,7 +27,6 @@ module Gitlab end chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger - chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware chain.add ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata chain.add ::Gitlab::SidekiqMiddleware::BatchLoader diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index ab126ea4749..d42bd672bac 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -282,7 +282,7 @@ module Gitlab Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } + Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall end end end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb deleted file mode 100644 index 0b38c98f710..00000000000 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class MemoryKiller - # Default the RSS limit to 0, meaning the MemoryKiller is disabled - MAX_RSS = (ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS'] || 0).to_s.to_i - # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit - GRACE_TIME = (ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || 15 * 60).to_s.to_i - # Wait 30 seconds for running jobs to finish during graceful shutdown - SHUTDOWN_WAIT = (ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || 30).to_s.to_i - - # Create a mutex used to ensure there will be only one thread waiting to - # shut Sidekiq down - MUTEX = Mutex.new - - attr_reader :worker - - def call(worker, job, queue) - yield - - @worker = worker - current_rss = get_rss - - return unless MAX_RSS > 0 && current_rss > MAX_RSS - - Thread.new do - # Return if another thread is already waiting to shut Sidekiq down - next unless MUTEX.try_lock - - warn("Sidekiq worker PID-#{pid} current RSS #{current_rss}"\ - " exceeds maximum RSS #{MAX_RSS} after finishing job #{worker.class} JID-#{job['jid']}") - - warn("Sidekiq worker PID-#{pid} will stop fetching new jobs"\ - " in #{GRACE_TIME} seconds, and will be shut down #{SHUTDOWN_WAIT} seconds later") - - # Wait `GRACE_TIME` to give the memory intensive job time to finish. - # Then, tell Sidekiq to stop fetching new jobs. - wait_and_signal(GRACE_TIME, 'SIGTSTP', 'stop fetching new jobs') - - # Wait `SHUTDOWN_WAIT` to give already fetched jobs time to finish. - # Then, tell Sidekiq to gracefully shut down by giving jobs a few more - # moments to finish, killing and requeuing them if they didn't, and - # then terminating itself. Sidekiq will replicate the TERM to all its - # children if it can. - wait_and_signal(SHUTDOWN_WAIT, 'SIGTERM', 'gracefully shut down') - - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't. - # Kill the whole pgroup, so we can be sure no children are left behind - wait_and_signal_pgroup(Sidekiq.options[:timeout] + 2, 'SIGKILL', 'die') - end - end - - private - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status == 0 - - output.to_i - end - - # If this sidekiq process is pgroup leader, signal to the whole pgroup - def wait_and_signal_pgroup(time, signal, explanation) - return wait_and_signal(time, signal, explanation) unless Process.getpgrp == pid - - warn("waiting #{time} seconds before sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) - sleep(time) - - warn("sending Sidekiq worker PGRP-#{pid} #{signal} (#{explanation})", signal: signal) - Process.kill(signal, 0) - end - - def wait_and_signal(time, signal, explanation) - warn("waiting #{time} seconds before sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) - sleep(time) - - warn("sending Sidekiq worker PID-#{pid} #{signal} (#{explanation})", signal: signal) - Process.kill(signal, pid) - end - - def pid - Process.pid - end - - def warn(message, signal: nil) - Sidekiq.logger.warn(class: worker.class.name, pid: pid, signal: signal, message: message) - end - end - end -end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 9d08d236720..17234bdf519 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -126,7 +126,7 @@ module Gitlab Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on - Sidekiq.redis { |redis| yield redis } + Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall end end private_class_method :with_redis diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index ef368767689..1527fd263e0 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -21,12 +21,16 @@ module Gitlab title = match[:title] description = match[:description].to_s.rstrip - issue = create_issue(title: title, description: description) + result = create_issue(title: title, description: description) - if issue.persisted? - presenter(issue).present + if result.success? + presenter(result[:issue]).present + elsif result[:issue] + presenter(result[:issue]).display_errors else - presenter(issue).display_errors + Gitlab::SlashCommands::Presenters::Error.new( + result.errors.join(', ') + ).message end end diff --git a/lib/gitlab/tracking/service_ping_context.rb b/lib/gitlab/tracking/service_ping_context.rb new file mode 100644 index 00000000000..393cd647e7f --- /dev/null +++ b/lib/gitlab/tracking/service_ping_context.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + class ServicePingContext + SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-0' + ALLOWED_SOURCES = %i[redis_hll].freeze + + def initialize(data_source:, event:) + unless ALLOWED_SOURCES.include?(data_source) + raise ArgumentError, "#{data_source} is not acceptable data source for ServicePingContext" + end + + @payload = { + data_source: data_source, + event_name: event + } + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(SCHEMA_URL, @payload) + end + end + end +end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index cf48aa49938..30f2efc8638 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -46,10 +46,7 @@ module Gitlab end def instrumentation_object - @instrumentation_object ||= instrumentation_class.constantize.new( - time_frame: definition.time_frame, - options: definition.attributes[:options] - ) + @instrumentation_object ||= instrumentation_class.constantize.new(definition.attributes) end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 2c50678c6bf..d6b1e62c84f 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -5,7 +5,7 @@ module Gitlab class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze InvalidError = Class.new(RuntimeError) diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 11e2fd22638..cd72f16d46d 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -13,62 +13,72 @@ module Gitlab end def all_time_data - aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) end def monthly_data - aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME) end def weekly_data - aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME)) + aggregated_metrics_data(Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME) + end + + def calculate_count_for_aggregation(aggregation:, time_frame:) + with_validate_configuration(aggregation, time_frame) do + source = SOURCES[aggregation[:source]] + + if aggregation[:operator] == UNION_OF_AGGREGATED_METRICS + source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + else + source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + end + end + rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + failure(error) end private attr_accessor :aggregated_metrics, :recorded_at - def aggregated_metrics_data(start_date:, end_date:, time_frame:) + def aggregated_metrics_data(time_frame) aggregated_metrics.each_with_object({}) do |aggregation, data| next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) next unless aggregation[:time_frame].include?(time_frame) - case aggregation[:source] - when REDIS_SOURCE - if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME - data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception( - DisallowedAggregationTimeFrame.new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") - ) - else - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) - end - when DATABASE_SOURCE - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) - else - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception(UnknownAggregationSource.new("Aggregation source: '#{aggregation[:source]}' must be included in #{SOURCES.keys}")) - - data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK - end + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, time_frame: time_frame) end end - def calculate_count_for_aggregation(aggregation:, start_date:, end_date:) - source = SOURCES[aggregation[:source]] - - case aggregation[:operator] - when UNION_OF_AGGREGATED_METRICS - source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) - when INTERSECTION_OF_AGGREGATED_METRICS - source.calculate_metrics_intersections(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) - else - Gitlab::ErrorTracking - .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) - Gitlab::Utils::UsageData::FALLBACK + def with_validate_configuration(aggregation, time_frame) + source = aggregation[:source] + + unless ALLOWED_METRICS_AGGREGATIONS.include?(aggregation[:operator]) + return failure( + UnknownAggregationOperator + .new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") + ) end - rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + + unless SOURCES[source] + return failure( + UnknownAggregationSource + .new("Aggregation source: '#{source}' must be included in #{SOURCES.keys}") + ) + end + + if time_frame == Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME && source == REDIS_SOURCE + return failure( + DisallowedAggregationTimeFrame + .new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") + ) + end + + yield + end + + def failure(error) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) Gitlab::Utils::UsageData::FALLBACK end @@ -82,6 +92,17 @@ module Gitlab def load_yaml_from_path(path) YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) end + + def time_constraints(time_frame) + case time_frame + when Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME + monthly_time_range + when Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME + weekly_time_range + when Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME + { start_date: nil, end_date: nil } + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb new file mode 100644 index 00000000000..63ead5a8cb0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: AggregatedMetric + # data_source: redis_hll + # options: + # aggregate: + # operator: OR + # attribute: user_id + # events: + # - 'incident_management_alert_status_changed' + # - 'incident_management_alert_assigned' + # - 'incident_management_alert_todo' + # - 'incident_management_alert_create_incident' + + class AggregatedMetric < BaseMetric + FALLBACK = -1 + + def initialize(metric_definition) + super + @source = parse_data_source_to_legacy_value(metric_definition) + @aggregate = options.fetch(:aggregate, {}) + end + + def value + alt_usage_data(fallback: FALLBACK) do + Aggregates::Aggregate + .new(Time.current) + .calculate_count_for_aggregation( + aggregation: aggregate_config, + time_frame: time_frame + ) + end + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) + end + + private + + attr_accessor :source, :aggregate + + # TODO: This method is a temporary measure that + # handles backwards compatibility until + # point 5 from is resolved https://gitlab.com/gitlab-org/gitlab/-/issues/370963#implementation + def parse_data_source_to_legacy_value(metric_definition) + return 'redis' if metric_definition[:data_source] == 'redis_hll' + + metric_definition[:data_source] + end + + def aggregate_config + { + source: source, + events: options[:events], + operator: aggregate[:operator] + } + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 5e20766b1b4..55da2315e45 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -23,9 +23,9 @@ module Gitlab attr_reader :metric_available end - def initialize(time_frame:, options: {}) - @time_frame = time_frame - @options = options + def initialize(metric_definition) + @time_frame = metric_definition.fetch(:time_frame) + @options = metric_definition.fetch(:options, {}) end def instrumentation diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index 67dc1455b23..642b67a3b02 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -7,7 +7,7 @@ module Gitlab class CountBulkImportsEntitiesMetric < DatabaseMetric operation :count - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super if source_type.present? && !source_type.in?(allowed_source_types) diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb index c5498ce530f..d485e8b4f72 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -7,7 +7,7 @@ module Gitlab class CountImportedProjectsMetric < DatabaseMetric operation :count - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super raise ArgumentError, "import_type options attribute is required" unless import_type.present? diff --git a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb new file mode 100644 index 00000000000..0c421dc3311 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DistinctCountProjectsWithExpirationPolicyDisabledMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + start { Project.minimum(:id) } + finish { Project.maximum(:id) } + + cache_start_and_finish_as :project_id + + relation { ::ContainerExpirationPolicy.where(enabled: false) } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 0f4b903b99c..7c646281598 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -28,9 +28,8 @@ module Gitlab end end - def initialize(time_frame: 'none', options: {}) - @time_frame = time_frame - @options = options + def initialize(metric_definition) + super(metric_definition.reverse_merge(time_frame: 'none')) end def value diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb new file mode 100644 index 00000000000..f22ee2aa3f5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_direct_installations_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabForJiraAppDirectInstallationsCountMetric < DatabaseMetric + operation :count + + relation { JiraConnectInstallation.direct_installations } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb new file mode 100644 index 00000000000..222a69faf8b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_for_jira_app_proxy_installations_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabForJiraAppProxyInstallationsCountMetric < DatabaseMetric + operation :count + + relation { JiraConnectInstallation.proxy_installations } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb b/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb new file mode 100644 index 00000000000..e2fdb3462c5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/merge_request_widget_extension_metric.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: MergeRequestWidgetExtensionMetric + # options: + # event: expand + # widget: terraform + # + class MergeRequestWidgetExtensionMetric < RedisMetric + extend ::Gitlab::Utils::Override + + def validate_options! + raise ArgumentError, "'event' option is required" unless metric_event.present? + raise ArgumentError, "'widget' option is required" unless widget_name.present? + end + + def widget_name + options[:widget] + end + + override :prefix + def prefix + 'i_code_review_merge_request_widget' + end + + private + + override :redis_key + def redis_key + "#{USAGE_PREFIX}#{prefix}_#{widget_name}_count_#{metric_event}".upcase + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index bb27cca1bb9..17009f7638e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -12,7 +12,7 @@ module Gitlab # events: # - g_analytics_valuestream # end - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super raise ArgumentError, "options events are required" unless metric_events.present? diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index 26d963e2407..ae3326fa845 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -17,12 +17,17 @@ module Gitlab include Gitlab::UsageDataCounters::RedisCounter USAGE_PREFIX = "USAGE_" + OPTIONS_PREFIX_KEY = :prefix - def initialize(time_frame:, options: {}) + def initialize(metric_definition) super + validate_options! + end + + def validate_options! raise ArgumentError, "'event' option is required" unless metric_event.present? - raise ArgumentError, "'prefix' option is required" unless prefix.present? + raise ArgumentError, "'prefix' option is required" unless options.has_key?(OPTIONS_PREFIX_KEY) end def metric_event @@ -30,7 +35,7 @@ module Gitlab end def prefix - options[:prefix] + options[OPTIONS_PREFIX_KEY] end def include_usage_prefix? @@ -50,9 +55,10 @@ module Gitlab private def redis_key - key = "#{prefix}_#{metric_event}".upcase + key = metric_event.dup + key.prepend("#{prefix}_") if prefix key.prepend(USAGE_PREFIX) if include_usage_prefix? - key + key.upcase end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb new file mode 100644 index 00000000000..d045265495a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/work_items_activity_aggregated_metric.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class WorkItemsActivityAggregatedMetric < AggregatedMetric + available? { Feature.enabled?(:track_work_items_activity) } + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e2232dc5e2a..87ccb9a31da 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -21,18 +21,18 @@ module Gitlab MAX_GENERATION_TIME_FOR_SAAS = 40.hours CE_MEMOIZED_VALUES = %i( - issue_minimum_id - issue_maximum_id - project_minimum_id - project_maximum_id - user_minimum_id - user_maximum_id - deployment_minimum_id - deployment_maximum_id - auth_providers - aggregated_metrics - recorded_at - ).freeze + issue_minimum_id + issue_maximum_id + project_minimum_id + project_maximum_id + user_minimum_id + user_maximum_id + deployment_minimum_id + deployment_maximum_id + auth_providers + aggregated_metrics + recorded_at + ).freeze class << self include Gitlab::Utils::UsageData @@ -346,7 +346,6 @@ module Gitlab start = minimum_id(Project) finish = maximum_id(Project) - results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable base = ::ContainerExpirationPolicy.active # rubocop: enable UsageData/LargeTable diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index eae1c593a8f..37c6e1af7c0 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -2,25 +2,24 @@ module Gitlab module UsageDataCounters - COUNTERS = [ - WikiPageCounter, - NoteCounter, - SnippetCounter, - SearchCounter, - CycleAnalyticsCounter, - ProductivityAnalyticsCounter, - SourceCodeCounter, - KubernetesAgentCounter, - MergeRequestWidgetExtensionCounter - ].freeze + COUNTERS = [].freeze COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ PackageEventCounter, MergeRequestCounter, DesignsCounter, DiffsCounter, + KubernetesAgentCounter, + NoteCounter, + SearchCounter, ServiceUsageDataCounter, - WebIdeCounter + WebIdeCounter, + WikiPageCounter, + SnippetCounter, + CycleAnalyticsCounter, + ProductivityAnalyticsCounter, + SourceCodeCounter, + MergeRequestWidgetExtensionCounter ].freeze UsageDataCounterError = Class.new(StandardError) @@ -28,13 +27,11 @@ module Gitlab class << self def unmigrated_counters - # we are using the #counters method instead of the COUNTERS const - # to make sure it's working correctly for `ee` version of UsageDataCounters - counters - self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + self::COUNTERS end def counters - self::COUNTERS + self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + unmigrated_counters + migrated_counters end def count(event_name) @@ -46,6 +43,12 @@ module Gitlab raise UnknownEvent, "Cannot find counter for event #{event_name}" end + + private + + def migrated_counters + COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + end end end end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 61c071c8738..1e8918c7c96 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -34,6 +34,21 @@ module Gitlab::UsageDataCounters Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml'))&.full_name end + def all_included_templates(template_name) + expanded_template_name = expand_template_name(template_name) + results = [expanded_template_name].tap do |result| + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) + data = YAML.safe_load(template.content, aliases: true) + [data['include']].compact.flatten.each do |ci_include| + if ci_include_template = ci_include['template'] + result.concat(all_included_templates(ci_include_template)) + end + end + end + + results.uniq.sort_by { _1['name'] } + end + private def template_to_event_name(template) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index f0cb9bcbe94..24a87ae01f4 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -19,18 +19,14 @@ module Gitlab ALLOWED_AGGREGATIONS = %i(daily weekly).freeze CATEGORIES_FOR_TOTALS = %w[ - analytics compliance - epics_usage error_tracking ide_edit - incident_management - issues_edit pipeline_authoring - quickactions ].freeze CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ + analytics ci_users deploy_token_packages code_review @@ -38,9 +34,13 @@ module Gitlab error_tracking ide_edit importer + incident_management incident_management_alerts + issues_edit kubernetes_agent + manage pipeline_authoring + quickactions search secure snippets diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 76c97a974d7..85524c766ca 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -10,54 +10,18 @@ category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly -- name: g_analytics_contribution - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_productivity - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_code_reviews - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly -- name: p_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly -- name: p_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - name: p_analytics_repo category: analytics redis_slot: analytics @@ -86,23 +50,3 @@ category: analytics redis_slot: analytics aggregation: weekly -- name: g_analytics_ci_cd_release_statistics - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_deployment_frequency - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_lead_time - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_time_to_restore_service - category: analytics - redis_slot: analytics - aggregation: weekly -- name: g_analytics_ci_cd_change_failure_rate - category: analytics - redis_slot: analytics - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 10e36a75a3a..5b80f6c6c0d 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -99,6 +99,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_coverage_fuzzing_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_dast_on_demand_api_scan category: ci_templates redis_slot: ci_templates @@ -499,27 +503,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_load_performance_testing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_secret_detection +- name: p_ci_templates_implicit_jobs_browser_performance_testing category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_license_scanning +- name: p_ci_templates_implicit_jobs_build category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -531,47 +519,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_license_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_build - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_dependency_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_test - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_iac - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_secret_detection_latest +- name: p_ci_templates_implicit_jobs_dast_default_branch_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -579,63 +527,35 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_jobs_deploy_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_browser_performance_testing_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_cf_provision - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_build_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_jobs_sast_iac_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast_runner_validation - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast_on_demand_scan +- name: p_ci_templates_implicit_jobs_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_secret_detection +- name: p_ci_templates_implicit_jobs_deploy_ec2 category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_license_scanning +- name: p_ci_templates_implicit_jobs_deploy_ecs category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_on_demand_api_scan +- name: p_ci_templates_implicit_jobs_helm_2to3 category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_coverage_fuzzing +- name: p_ci_templates_implicit_jobs_license_scanning category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_api_fuzzing_latest +- name: p_ci_templates_implicit_jobs_sast category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_secure_binaries +- name: p_ci_templates_implicit_jobs_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_api +- name: p_ci_templates_implicit_jobs_test category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -643,11 +563,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_sast_iac +- name: p_ci_templates_implicit_security_dast category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -655,27 +571,15 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_dast_api_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_container_scanning_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_api_fuzzing - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_implicit_security_dast +- name: p_ci_templates_implicit_security_license_scanning category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_fortify_fod_sast +- name: p_ci_templates_implicit_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_sast_iac_latest +- name: p_ci_templates_implicit_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 29b231f88f8..c13c7657576 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -84,11 +84,6 @@ redis_slot: incident_management category: incident_management aggregation: weekly -- name: incident_management_incident_published - redis_slot: incident_management - category: incident_management - aggregation: weekly - feature_flag: usage_data_incident_management_incident_published - name: incident_management_incident_relate redis_slot: incident_management category: incident_management @@ -114,29 +109,11 @@ redis_slot: incident_management category: incident_management aggregation: weekly -# Incident management linked resources -- name: incident_management_issuable_resource_link_created - redis_slot: incident_management - category: incident_management - aggregation: weekly -- name: incident_management_issuable_resource_link_deleted - redis_slot: incident_management - category: incident_management - aggregation: weekly -- name: incident_management_issuable_resource_link_visited - redis_slot: incident_management - category: incident_management - aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident redis_slot: incident_management category: incident_management_alerts aggregation: weekly -# Incident management on-call -- name: i_incident_management_oncall_notification_sent - redis_slot: incident_management - category: incident_management_oncall - aggregation: weekly # Testing category - name: i_testing_test_case_parsed category: testing @@ -150,7 +127,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_ci_i_testing_test_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -192,14 +168,6 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_iteration_changed - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_weight_changed - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management @@ -228,18 +196,6 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_added_to_epic - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_removed_from_epic - category: issues_edit - redis_slot: project_management - aggregation: daily -- name: g_project_management_issue_changed_epic - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management @@ -276,20 +232,11 @@ category: issues_edit redis_slot: project_management aggregation: daily -- name: g_project_management_issue_health_status_changed - category: issues_edit - redis_slot: project_management - aggregation: daily - name: g_project_management_issue_cloned category: issues_edit redis_slot: project_management aggregation: daily # Secrets Management -- name: i_ci_secrets_management_vault_build_created - category: ci_secrets_management - redis_slot: ci_secrets_management - aggregation: weekly - feature_flag: usage_data_i_ci_secrets_management_vault_build_created - name: i_snippets_show category: snippets redis_slot: snippets @@ -342,11 +289,6 @@ category: geo redis_slot: geo aggregation: daily -# Growth -- name: users_clicking_registration_features_offer - category: growth - redis_slot: users - aggregation: weekly # Manage - name: unique_active_user category: manage diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml deleted file mode 100644 index dd6625a9cc9..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ /dev/null @@ -1,227 +0,0 @@ -# Epic events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_epic_created - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# content change events - -- name: project_management_users_unchecking_epic_task - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: project_management_users_checking_epic_task - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_titles - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_descriptions - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# epic notes - -- name: g_project_management_users_creating_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_destroying_epic_notes - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# emoji - -- name: g_project_management_users_awarding_epic_emoji - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_removing_epic_emoji - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# start date events - -- name: g_project_management_users_setting_epic_start_date_as_fixed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_fixed_epic_start_date - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_start_date_as_inherited - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# due date events - -- name: g_project_management_users_setting_epic_due_date_as_fixed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_fixed_epic_due_date - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_due_date_as_inherited - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -# relationships - -- name: g_project_management_epic_issue_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_issue_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_issue_moved_from_project - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_updating_epic_parent - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_closed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_reopened - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: 'g_project_management_issue_promoted_to_epic' - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_confidential - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_setting_epic_visible - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_users_changing_labels - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_destroyed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_cross_referenced - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_users_epic_issue_added_from_epic - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_related_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_related_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocking_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocking_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocked_added - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - -- name: g_project_management_epic_blocked_removed - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity - diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 58a0c0695af..69b348b9a22 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,17 +1,17 @@ --- -- name: i_quickactions_approve +- name: i_quickactions_assign_multiple category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_unapprove +- name: i_quickactions_approve category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_assign_single +- name: i_quickactions_unapprove category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_assign_multiple +- name: i_quickactions_assign_single category: quickactions redis_slot: quickactions aggregation: weekly @@ -31,18 +31,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_child_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_clear_weight - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_clear_health_status - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_clone category: quickactions redis_slot: quickactions @@ -83,18 +71,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_estimate category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_iteration - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_label category: quickactions redis_slot: quickactions @@ -115,14 +95,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_parent_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_promote - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_promote_to_incident category: quickactions redis_slot: quickactions @@ -131,14 +103,6 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_page - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_publish - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_ready category: quickactions redis_slot: quickactions @@ -163,34 +127,18 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_child_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_due_date category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_estimate category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_iteration - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_milestone category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_remove_parent_epic - category: quickactions - redis_slot: quickactions - aggregation: weekly - name: i_quickactions_remove_time_spent category: quickactions redis_slot: quickactions @@ -275,19 +223,15 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_weight - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_health_status +- name: i_quickactions_wip category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_wip +- name: i_quickactions_zoom category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_zoom +- name: i_quickactions_link category: quickactions redis_slot: quickactions aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index 6cd7836ea94..ee828fc0f72 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -14,3 +14,16 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_work_item_labels + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity +- name: users_updating_work_item_iteration + # The event tracks an EE feature. + # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. + # It will report 0 for CE instances and should not be used with 'AND' aggregators. + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index 99b4c082310..a0fd04596fc 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -6,6 +6,7 @@ module Gitlab WORK_ITEM_CREATED = 'users_creating_work_items' WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' WORK_ITEM_DATE_CHANGED = 'users_updating_work_item_dates' + WORK_ITEM_LABELS_CHANGED = 'users_updating_work_item_labels' class << self def track_work_item_created_action(author:) @@ -20,6 +21,10 @@ module Gitlab track_unique_action(WORK_ITEM_DATE_CHANGED, author) end + def track_work_item_labels_changed_action(author:) + track_unique_action(WORK_ITEM_LABELS_CHANGED, author) + end + private def track_unique_action(action, author) diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c2983779603..4486ca53966 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -75,10 +75,6 @@ module Gitlab } end - def epics_deepest_relationship_level - { epics_deepest_relationship_level: 0 } - end - def topology_usage_data { duration_s: 0, diff --git a/lib/gitlab/utils/execution_tracker.rb b/lib/gitlab/utils/execution_tracker.rb index 6d48658853c..92398926e1b 100644 --- a/lib/gitlab/utils/execution_tracker.rb +++ b/lib/gitlab/utils/execution_tracker.rb @@ -3,7 +3,7 @@ module Gitlab module Utils class ExecutionTracker - MAX_RUNTIME = 30.seconds + MAX_RUNTIME = 60.seconds ExecutionTimeOutError = Class.new(StandardError) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 19bdeefed7e..0b818b99ac7 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -291,14 +291,6 @@ module Gitlab end end - def epics_deepest_relationship_level - with_duration do - # rubocop: disable UsageData/LargeTable - { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } - # rubocop: enable UsageData/LargeTable - end - end - private def prometheus_client(verify:) diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb index 349c7a020cc..8c6de56292a 100644 --- a/lib/gitlab/web_hooks.rb +++ b/lib/gitlab/web_hooks.rb @@ -3,5 +3,6 @@ module Gitlab module WebHooks GITLAB_EVENT_HEADER = 'X-Gitlab-Event' + GITLAB_INSTANCE_HEADER = 'X-Gitlab-Instance' end end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 8acbfc144e9..f8a6980f208 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -37,7 +37,7 @@ module Gitlab !verified_signature || user.nil? - if user.verified_emails.include?(@email) && certificate_email == @email + if user.verified_emails.include?(@email.downcase) && certificate_email.casecmp?(@email) :verified else :unverified |