diff options
Diffstat (limited to 'lib/gitlab/ci')
58 files changed, 774 insertions, 1036 deletions
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index b1dee0e1ecc..466706384c0 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -77,7 +77,7 @@ module Gitlab end def set_section_duration(duration) - @section_duration = Time.at(duration.to_i).strftime('%M:%S') + @section_duration = Time.at(duration.to_i).utc.strftime('%M:%S') end def flush_current_segment! diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 7589fa5ff8b..96702420e9d 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -24,26 +24,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text @status ? ("%.2f%%" % @status) : 'unknown' end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width @status ? 54 : 58 end diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index 8430b01fc9a..c39f96e4a34 100644 --- a/lib/gitlab/ci/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -28,26 +28,10 @@ module Gitlab::Ci @key_width = badge.customization.dig(:key_width) end - def key_text - if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE - @key_text - else - @entity.to_s - end - end - def value_text STATUS_RENAME[@status.to_s] || @status.to_s end - def key_width - if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) - @key_width - else - 62 - end - end - def value_width 54 end diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 0580dad72ba..d514a8577bd 100644 --- a/lib/gitlab/ci/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -8,6 +8,7 @@ module Gitlab::Ci class Template MAX_KEY_TEXT_SIZE = 64 MAX_KEY_WIDTH = 512 + DEFAULT_KEY_WIDTH = 62 def initialize(badge) @entity = badge.entity @@ -15,7 +16,11 @@ module Gitlab::Ci end def key_text - raise NotImplementedError + if @key_text && @key_text.size <= MAX_KEY_TEXT_SIZE + @key_text + else + @entity.to_s + end end def value_text @@ -23,7 +28,11 @@ module Gitlab::Ci end def key_width - raise NotImplementedError + if @key_width && @key_width.between?(1, MAX_KEY_WIDTH) + @key_width + else + DEFAULT_KEY_WIDTH + end end def value_width diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index e6ef12975c2..b98d1d7b330 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -7,6 +7,11 @@ class Gitlab::Ci::Build::AutoRetry scheduler_failure: 2 }.freeze + RETRY_OVERRIDES = { + ci_quota_exceeded: 0, + no_matching_runner: 0 + }.freeze + def initialize(build) @build = build end @@ -19,13 +24,18 @@ class Gitlab::Ci::Build::AutoRetry private + delegate :failure_reason, to: :@build + def within_max_retry_limit? max_allowed_retries > 0 && max_allowed_retries > @build.retries_count end def max_allowed_retries strong_memoize(:max_allowed_retries) do - options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0) + RETRY_OVERRIDES[failure_reason.to_sym] || + options_retry_max || + DEFAULT_RETRIES[failure_reason.to_sym] || + 0 end end @@ -38,7 +48,7 @@ class Gitlab::Ci::Build::AutoRetry end def retry_on_reason_or_always? - options_retry_when.include?(@build.failure_reason.to_s) || + options_retry_when.include?(failure_reason.to_s) || options_retry_when.include?('always') end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index 29dc48c7b42..f1b67635c08 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -35,14 +35,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: @config, - artifacts: true, - optional: false } - else - { name: @config, - artifacts: true } - end + { name: @config, + artifacts: true, + optional: false } end end @@ -66,14 +61,9 @@ module Gitlab end def value - if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) - { name: job, - artifacts: artifacts || artifacts.nil?, - optional: !!optional } - else - { name: job, - artifacts: artifacts || artifacts.nil? } - end + { name: job, + artifacts: artifacts || artifacts.nil?, + optional: !!optional } end end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 947b6787aa0..79dfb0eec1d 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -98,7 +98,6 @@ module Gitlab def validate_against_warnings # If rules are valid format and workflow rules are not specified return unless rules_value - return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning? last_rule = rules_value.last diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f2fd8ac7fd9..4db25fb0930 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast secret_detection dependency_scanning container_scanning - dast performance browser_performance load_performance license_management license_scanning metrics lsif + dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications requirements coverage_fuzzing api_fuzzing].freeze @@ -36,7 +36,6 @@ module Gitlab validates :performance, array_of_strings_or_string: true validates :browser_performance, array_of_strings_or_string: true validates :load_performance, array_of_strings_or_string: true - validates :license_management, array_of_strings_or_string: true validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true @@ -44,7 +43,7 @@ module Gitlab validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true - validates :cluster_applications, array_of_strings_or_string: true + validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index a8f78b62d8d..e6ff33d6f79 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -28,11 +28,6 @@ module Gitlab end end - def matching? - super && - Feature.enabled?(:ci_dynamic_child_pipeline, project, default_enabled: true) - end - private def project diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index c4b4a7a0a73..47441fa3818 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -6,7 +6,7 @@ module Gitlab module External module File class Template < Base - attr_reader :location, :project + attr_reader :location SUFFIX = '.gitlab-ci.yml' @@ -41,7 +41,7 @@ module Gitlab end def fetch_template_content - Gitlab::Template::GitlabCiYmlTemplate.find(template_name, project)&.content + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index efd48a9b29f..bc03658aab8 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,6 +6,10 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' VALID_SYNTAX_SAMPLE_CRON = '* * * * *' + def self.parse_natural(expression, cron_timezone = 'UTC') + new(Fugit::Nat.parse(expression)&.original, cron_timezone) + end + def initialize(cron, cron_timezone = 'UTC') @cron = cron @cron_timezone = timezone_name(cron_timezone) @@ -27,6 +31,10 @@ module Gitlab try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? end + def match?(time) + cron_line.match?(time) + end + private def timezone_name(timezone) diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c8e4d9ed763..fe69a170404 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,14 +18,6 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - def self.merge_base_pipeline_for_metrics_comparison?(project) - Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) - end - - def self.raise_job_rules_without_workflow_rules_warning? - ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -33,10 +25,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.trace_overwrite? - ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) - end - def self.accept_trace?(project) ::Feature.enabled?(:ci_enable_live_trace, project) && ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) @@ -53,10 +41,6 @@ module Gitlab def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end - - def self.background_pipeline_retry_endpoint?(project) - ::Feature.enabled?(:background_pipeline_retry_endpoint, project) - end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0b94debb24e..3fb86b8b3e8 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -54,6 +54,7 @@ module Gitlab user_login: user&.username, user_email: user&.email, pipeline_id: build.pipeline.id.to_s, + pipeline_source: build.pipeline.source.to_s, job_id: build.id.to_s, ref: source_ref, ref_type: ref_type, diff --git a/lib/gitlab/ci/matching/build_matcher.rb b/lib/gitlab/ci/matching/build_matcher.rb new file mode 100644 index 00000000000..dff7d9141d9 --- /dev/null +++ b/lib/gitlab/ci/matching/build_matcher.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + class BuildMatcher + ATTRIBUTES = %i[ + protected + tag_list + build_ids + project + ].freeze + + attr_reader(*ATTRIBUTES) + alias_method :protected?, :protected + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def has_tags? + tag_list.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/matching/runner_matcher.rb b/lib/gitlab/ci/matching/runner_matcher.rb new file mode 100644 index 00000000000..63642674936 --- /dev/null +++ b/lib/gitlab/ci/matching/runner_matcher.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Matching + ### + # This class is used to check if a build can be picked by a runner: + # + # runner = Ci::Runner.find(id) + # build = Ci::Build.find(id) + # runner.runner_matcher.matches?(build.build_matcher) + # + # There are also class level methods to build matchers: + # + # `project.builds.build_matchers(project)` returns a distinct collection + # of build matchers. + # `Ci::Runner.runner_matchers` returns a distinct collection of runner matchers. + # + class RunnerMatcher + ATTRIBUTES = %i[ + runner_type + public_projects_minutes_cost_factor + private_projects_minutes_cost_factor + run_untagged + access_level + tag_list + ].freeze + + attr_reader(*ATTRIBUTES) + + def initialize(params) + ATTRIBUTES.each do |attribute| + instance_variable_set("@#{attribute}", params.fetch(attribute)) + end + end + + def matches?(build_matcher) + ensure_build_matcher_instance!(build_matcher) + return false if ref_protected? && !build_matcher.protected? + + accepting_tags?(build_matcher) + end + + def instance_type? + runner_type.to_sym == :instance_type + end + + private + + def ref_protected? + access_level.to_sym == :ref_protected + end + + def accepting_tags?(build_matcher) + (run_untagged || build_matcher.has_tags?) && (build_matcher.tag_list - tag_list).empty? + end + + def ensure_build_matcher_instance!(build_matcher) + return if build_matcher.is_a?(Matching::BuildMatcher) + + raise ArgumentError, 'only Gitlab::Ci::Matching::BuildMatcher are allowed' + end + end + end + end +end + +Gitlab::Ci::Matching::RunnerMatcher.prepend_mod_with('Gitlab::Ci::Matching::RunnerMatcher') diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index ca7fbde6713..364ae66844e 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -69,6 +69,7 @@ module Gitlab elsif data.key?('error') status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR system_output = data['error'] + attachment = attachment_path(data['system_out']) elsif data.key?('skipped') status = ::Gitlab::Ci::Reports::TestCase::STATUS_SKIPPED system_output = data['skipped'] diff --git a/lib/gitlab/ci/pipeline/chain/validate/after_config.rb b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb new file mode 100644 index 00000000000..c3db00b4fb2 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/after_config.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class AfterConfig < Chain::Base + include Chain::Helpers + + def perform! + end + + def break? + @pipeline.errors.any? + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::AfterConfig') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 539b44513f0..27bb7fdc05a 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -12,12 +12,9 @@ module Gitlab DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 ACCEPTED_STATUS = 200 - DOT_COM_REJECTED_STATUS = 406 - GENERAL_REJECTED_STATUS = (400..499).freeze + REJECTED_STATUS = 406 def perform! - return unless enabled? - pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' @@ -32,24 +29,17 @@ module Gitlab private - def enabled? - return true unless Gitlab.com? - - ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) - end - def validate_external return true unless validation_service_url # 200 - accepted - # 406 - not accepted on GitLab.com - # 4XX - not accepted for other installations + # 406 - rejected # everything else - accepted and logged response_code = validate_service_request.code case response_code when ACCEPTED_STATUS true - when rejected_status + when REJECTED_STATUS false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" @@ -60,14 +50,6 @@ module Gitlab true end - def rejected_status - if Gitlab.com? - DOT_COM_REJECTED_STATUS - else - GENERAL_REJECTED_STATUS - end - end - def validate_service_request headers = { 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, @@ -107,7 +89,9 @@ module Gitlab id: current_user.id, username: current_user.username, email: current_user.email, - created_at: current_user.created_at&.iso8601 + created_at: current_user.created_at&.iso8601, + current_sign_in_ip: current_user.current_sign_in_ip, + last_sign_in_ip: current_user.last_sign_in_ip }, pipeline: { sha: pipeline.sha, diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb index 7befc126ca9..31ddf2c4241 100644 --- a/lib/gitlab/ci/pipeline/preloader.rb +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -20,6 +20,7 @@ module Gitlab preloader.preload_ref_commits preloader.preload_pipeline_warnings preloader.preload_stages_warnings + preloader.preload_persisted_environments end end end @@ -54,6 +55,13 @@ module Gitlab def preload_stages_warnings @pipeline.stages.each { |stage| stage.number_of_warnings } end + + # This batch loads the associated environments of multiple actions (builds) + # that can't use `preload` due to the indirect relationship. + def preload_persisted_environments + @pipeline.scheduled_actions.each { |action| action.persisted_environment } + @pipeline.manual_actions.each { |action| action.persisted_environment } + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 39dee7750d6..299b27a5f13 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -146,7 +146,7 @@ module Gitlab end @needs_attributes.flat_map do |need| - next if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) && need[:optional] + next if need[:optional] result = @previous_stages.any? do |stage| stage.seeds_names.include?(need[:name]) diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 46e4373ec85..859aeb35f26 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -20,6 +20,8 @@ module Gitlab :build_can_pick, :build_not_pick, :build_not_pending, + :build_queue_push, + :build_queue_pop, :build_temporary_locked, :build_conflict_lock, :build_conflict_exception, @@ -31,7 +33,9 @@ module Gitlab :queue_replication_lag, :runner_pre_assign_checks_failed, :runner_pre_assign_checks_success, - :runner_queue_tick + :runner_queue_tick, + :shared_runner_build_new, + :shared_runner_build_done ].to_set.freeze QUEUE_DEPTH_HISTOGRAMS = [ @@ -77,11 +81,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def increment_queue_operation(operation) - if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) - raise ArgumentError, "unknown queue operation: #{operation}" - end - - self.class.queue_operations_total.increment(operation: operation) + self.class.increment_queue_operation(operation) end def observe_queue_depth(queue, size) @@ -121,6 +121,14 @@ module Gitlab result end + def self.increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + queue_operations_total.increment(operation: operation) + end + def self.observe_active_runners(runners_proc) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 239fc3b15e7..287a03cefe2 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -8,6 +8,7 @@ module Gitlab DEFAULT_MAX_TESTS = 100 DEFAULT_MIN_TESTS = 10 + TestSummary = Struct.new(:new_failures, :existing_failures, :resolved_failures, :new_errors, :existing_errors, :resolved_errors, keyword_init: true) attr_reader :name, :base_suite, :head_suite @@ -90,7 +91,7 @@ module Gitlab def limited_tests strong_memoize(:limited_tests) do # rubocop: disable CodeReuse/ActiveRecord - OpenStruct.new( + TestSummary.new( new_failures: new_failures.take(max_tests), existing_failures: existing_failures.take(max_tests(new_failures)), resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)), diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index cbd72f54ff4..66f51f63585 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -30,7 +30,8 @@ module Gitlab reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', - ci_quota_exceeded: 'no more CI minutes available' + ci_quota_exceeded: 'no more CI minutes available', + no_matching_runner: 'no matching runner available' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index a13f2046291..5680950bba8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -11,7 +11,7 @@ # * test: TEST_DISABLED # * code_quality: CODE_QUALITY_DISABLED # * license_management: LICENSE_MANAGEMENT_DISABLED -# * performance: PERFORMANCE_DISABLED +# * browser_performance: BROWSER_PERFORMANCE_DISABLED # * load_performance: LOAD_PERFORMANCE_DISABLED # * sast: SAST_DISABLED # * secret_detection: SECRET_DETECTION_DISABLED diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 4dc88418671..07d0de5f9e5 100644 --- a/lib/gitlab/ci/templates/Getting-started.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -16,7 +16,7 @@ build-job: # This job runs in the build stage, which runs first. stage: build script: - echo "Compiling the code..." - - echo "Compile complete. + - echo "Compile complete." unit-test-job: # This job runs in the test stage. stage: test # It only starts when the job in the build stage completes successfully. diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 01907ef9e2e..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html -performance: +browser_performance: stage: performance image: docker:19.03.12 allow_failure: true @@ -72,6 +72,6 @@ performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 5216a46745c..56899614cc6 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -72,6 +72,6 @@ browser_performance: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$PERFORMANCE_DISABLED' + - if: '$BROWSER_PERFORMANCE_DISABLED' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index abcb347b146..cf99d722e4d 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0' variables: - DOCKER_TLS_CERTDIR: "" + DOCKER_TLS_CERTDIR: '' services: - - name: "docker:20.10.6-dind" + - name: 'docker:20.10.6-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b29342216fc..48e877684f6 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7ad5a9e2bba..00fcfa64a18 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 29edada4041..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.7" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml new file mode 100644 index 00000000000..6af79728dc8 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -0,0 +1,335 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + + SAST_EXCLUDED_ANALYZERS: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + SCAN_KUBERNETES_MANIFESTS: "false" + +sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + variables: + SEARCH_MAX_DEPTH: 4 + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + +.sast-analyzer: + extends: sast + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +bandit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + +brakeman-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.rb' + - '**/Gemfile' + +eslint-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +flawfinder-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.c' + - '**/*.cpp' + +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never + - if: $CI_COMMIT_BRANCH && + $SCAN_KUBERNETES_MANIFESTS == 'true' + +gosec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.go' + +.mobsf-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + +mobsf-android-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + +nodejs-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/package.json' + +phpcs-security-audit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.php' + +pmd-apex-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.cls' + +security-code-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.csproj' + - '**/*.vbproj' + +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +sobelow-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never + - if: $CI_COMMIT_BRANCH + exists: + - 'mix.exs' + +spotbugs-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never + - if: $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml new file mode 100644 index 00000000000..d0595491400 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECRETS_ANALYZER_VERSION: "3" + SECRET_DETECTION_EXCLUDED_PATHS: "" + +.secret-analyzer: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + +secret_detection: + extends: .secret-analyzer + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi + - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - /analyzer run + - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 6f30fc2dcd5..ca63e942130 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,21 @@ +################################################################################ +# WARNING +################################################################################ +# +# This template is DEPRECATED and scheduled for removal in GitLab 15.0 +# See https://gitlab.com/gitlab-org/gitlab/-/issues/333610 for more context. +# +# To get started with a Cluster Management Project, we instead recommend +# using the updated project template: +# +# - Documentation: https://docs.gitlab.com/ee/user/clusters/management_project_template.html +# - Source code: https://gitlab.com/gitlab-org/project-templates/cluster-management/ +# +################################################################################ + apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.43.1" environment: name: production variables: @@ -9,11 +24,9 @@ apply: script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: - refs: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: - reports: - cluster_applications: gl-cluster-applications.json when: on_failure paths: - tiller.log diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 275364afae4..1bdaaeede43 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,6 +1,6 @@ # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/ruby/tags/ -image: "ruby:2.5" +image: ruby:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 90fad1550ff..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,279 +1,33 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables - -stages: - - build - - test - - deploy - - fuzz +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: + FUZZAPI_VERSION: "1" SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: "1.6" - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:5000 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:5000 - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 5000:5000 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:5000 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 5001:5000 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8fa33026011..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: FUZZAPI_VERSION: "1" diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index c628e30b2c7..bd163f9db94 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,60 +1,44 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the DOCKER_IMAGE variable. If DOCKER_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `DOCKER_USER` and `DOCKER_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - CS_MAJOR_VERSION: 3 + CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 -.cs_common: - stage: test +container_scanning: image: "$CS_ANALYZER_IMAGE" + stage: test variables: - # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` - # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template - # for details + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting GIT_STRATEGY: none - # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true artifacts: reports: container_scanning: gl-container-scanning-report.json + paths: [gl-container-scanning-report.json] dependencies: [] - -container_scanning: - extends: .cs_common - variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" - CS_PROJECT: 'klar' - services: - - name: $CLAIR_DB_IMAGE - alias: clair-vulnerabilities-db - script: - - /analyzer run - rules: - - if: $CONTAINER_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION =~ /^[0-3]$/ - -container_scanning_new: - extends: .cs_common - variables: - CS_PROJECT: 'container-scanning' script: - gtcs scan - artifacts: - paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && - $CS_MAJOR_VERSION !~ /^[0-3]$/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 9d47537c0f0..2dbfb80b419 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,5 +1,8 @@ # Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables + variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index b40c4e982f7..9170e943e9d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -13,9 +13,8 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) -# List of variables available to configure the DAST API scanning tool: -# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: # Setting this variable affects all Security templates diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 7abecfb7e49..a2b112b8e9f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables stages: - build @@ -11,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index b6282da18a4..6834766da3d 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -13,12 +13,11 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" @@ -47,10 +46,13 @@ dast: $REVIEW_DISABLED && $DAST_WEBSITE == null && $DAST_API_SPECIFICATION == null when: never - - if: $CI_COMMIT_BRANCH && + - if: $CI_MERGE_REQUEST_IID && $CI_KUBERNETES_ACTIVE && $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_MERGE_REQUEST_IID && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) + - if: $CI_OPEN_MERGE_REQUESTS + when: never - if: $CI_COMMIT_BRANCH && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $DAST_API_SPECIFICATION + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && ($DAST_WEBSITE || $DAST_API_SPECIFICATION) diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 53d68c24d26..8df5ce79fe8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml deleted file mode 100644 index 87f78d0c887..00000000000 --- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624 -# Please, use License-Scanning.gitlab-ci.yml template instead - -include: - - template: License-Scanning.gitlab-ci.yml - -license_scanning: - before_script: - - | - echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template. - Please replace it with the License-Scanning.gitlab-ci.yml template instead. - For more details visit - https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 21e926ef275..870684c9f1d 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a8d45e80356..77ce813dd4f 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -1,340 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/SAST.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - # Setting this variable will affect all Security templates - # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" - SAST_EXCLUDED_ANALYZERS: "" - SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" - SAST_ANALYZER_IMAGE_TAG: 2 - SCAN_KUBERNETES_MANIFESTS: "false" - -sast: - stage: test - artifacts: - reports: - sast: gl-sast-report.json - rules: - - when: never - variables: - SEARCH_MAX_DEPTH: 4 - script: - - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" - - exit 1 - -.sast-analyzer: - extends: sast - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - script: - - /analyzer run - -bandit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /bandit/ - exists: - - '**/*.py' - -brakeman-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /brakeman/ - exists: - - '**/*.rb' - - '**/Gemfile' - -eslint-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /eslint/ - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -flawfinder-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ - exists: - - '**/*.c' - - '**/*.cpp' - -kubesec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && - $SCAN_KUBERNETES_MANIFESTS == 'true' - -gosec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /gosec/ - exists: - - '**/*.go' - -.mobsf-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - -mobsf-android-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.apk' - - '**/AndroidManifest.xml' - -mobsf-ios-sast: - extends: .mobsf-sast - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/*.ipa' - - '**/*.xcodeproj/*' - -nodejs-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ - exists: - - '**/package.json' - -phpcs-security-audit-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ - exists: - - '**/*.php' - -pmd-apex-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ - exists: - - '**/*.cls' - -security-code-scan-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ - exists: - - '**/*.csproj' - - '**/*.vbproj' - -semgrep-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ - exists: - - '**/*.py' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - -sobelow-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /sobelow/ - exists: - - 'mix.exs' - -spotbugs-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ - when: never - - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' - exists: - - '**/AndroidManifest.xml' - when: never - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ - exists: - - '**/*.groovy' - - '**/*.java' - - '**/*.scala' - - '**/*.kt' +include: + template: Jobs/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index c255fb4707a..d4ea7165d0a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,45 +1,5 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection -# -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# This template moved to Jobs/Secret-Detection.gitlab-ci.yml in GitLab 14.0 +# Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 -variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SECRETS_ANALYZER_VERSION: "3" - SECRET_DETECTION_EXCLUDED_PATHS: "" - - -.secret-analyzer: - stage: test - image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" - services: [] - allow_failure: true - # `rules` must be overridden explicitly by each child job - # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 - artifacts: - reports: - secret_detection: gl-secret-detection-report.json - -secret_detection_default_branch: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH - script: - - /analyzer run - -secret_detection: - extends: .secret-analyzer - rules: - - if: $SECRET_DETECTION_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - script: - - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - - /analyzer run - - rm "$CI_COMMIT_SHA"_commit_list.txt +include: + template: Jobs/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index ac975fbbeab..d410c49b9a4 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -15,7 +15,6 @@ variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, - klar, clair-vulnerabilities-db, license-finder, dast, api-fuzzing @@ -78,6 +77,8 @@ brakeman: gosec: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -161,28 +162,6 @@ kubesec: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ -# -# Container Scanning jobs -# - -klar: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ - variables: - SECURE_BINARIES_ANALYZER_VERSION: "3" - -clair-vulnerabilities-db: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ - variables: - SECURE_BINARIES_IMAGE: arminc/clair-db - SECURE_BINARIES_ANALYZER_VERSION: latest # # Dependency Scanning jobs diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 6b9db1c2e0f..62b32d7c2db 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,61 +1,22 @@ -# Official image for Hashicorp's Terraform. It uses light image which is Alpine -# based as it is much lighter. -# -# Entrypoint is also needed as image by default set `terraform` binary as an -# entrypoint. -image: - name: registry.gitlab.com/gitlab-org/gitlab-build-images:terraform - entrypoint: - - '/usr/bin/env' - - 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' - -# Default output file for Terraform plan -variables: - PLAN: plan.tfplan - JSON_PLAN_FILE: tfplan.json - -cache: - paths: - - .terraform - - .terraform.lock.hcl - -before_script: - - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" - - terraform --version - - terraform init +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: + - init - validate - build - - test - deploy +init: + extends: .init + validate: - stage: validate - script: - - terraform validate + extends: .validate -plan: - stage: build - script: - - terraform plan -out=$PLAN - - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" - artifacts: - paths: - - $PLAN - reports: - terraform: $JSON_PLAN_FILE +build: + extends: .build -# Separate apply job for manual launching Terraform as it can be destructive -# action. -apply: - stage: deploy - environment: - name: production - script: - - terraform apply -input=false $PLAN +deploy: + extends: .deploy dependencies: - - plan - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: manual + - build diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 404d4a4c6db..f0621165f8a 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -6,7 +6,7 @@ stages: - deploy - performance -performance: +browser_performance: stage: performance image: docker:git variables: diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 035ba52da84..536cf9bd8d8 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -1,22 +1,28 @@ -default: +publish: image: node:latest - - # Validate that the repository contains a package.json and extract a few values from it. - before_script: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ + changes: + - package.json + script: + # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - | - if [[ ! -f package.json ]]; then - echo "No package.json found! A package.json file is required to publish a package to GitLab's NPM registry." - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#creating-a-project' - exit 1 + if [[ ! -f .npmrc ]]; then + echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' + { + echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" + echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" + } >> .npmrc fi + - echo "Created the following .npmrc:"; cat .npmrc + + # Extract a few values from package.json - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") -# Validate that the package name is properly scoped to the project's root namespace. -# For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention -validate_package_scope: - stage: build - script: + # Validate that the package name is properly scoped to the project's root namespace. + # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - | if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" @@ -24,36 +30,12 @@ validate_package_scope: exit 1 fi -# If no .npmrc if included in the repo, generate a temporary one to use during the publish step -# that is configured to publish to GitLab's NPM registry -create_npmrc: - stage: build - script: + # Compare the version in package.json to all published versions. + # If the package.json version has not yet been published, run `npm publish`. - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#authenticating-with-a-ci-job-token' - - { - echo '@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/packages/npm/:_authToken=${CI_JOB_TOKEN}' - echo '//${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}' - } >> .npmrc - - fi - artifacts: - paths: - - .npmrc - -# Publish the package. If the version in package.json has not yet been published, it will be -# published to GitLab's NPM registry. If the version already exists, the publish command -# will fail and the existing package will not be updated. -publish_package: - stage: deploy - script: - - | - { - npm publish && + if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + npm publish echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - } || { - echo "No new version of ${NPM_PACKAGE_NAME} published. This is most likely because version ${NPM_PACKAGE_VERSION} already exists in GitLab's NPM registry." - } + else + echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." + fi diff --git a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml deleted file mode 100644 index 536cf9bd8d8..00000000000 --- a/lib/gitlab/ci/templates/npm.latest.gitlab-ci.yml +++ /dev/null @@ -1,41 +0,0 @@ -publish: - image: node:latest - stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_REF_NAME =~ /^v\d+\.\d+\.\d+.*$/ - changes: - - package.json - script: - # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - - | - if [[ ! -f .npmrc ]]; then - echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' - { - echo "@${CI_PROJECT_ROOT_NAMESPACE}:registry=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" - echo "${CI_API_V4_URL#http*:}/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=\${CI_JOB_TOKEN}" - } >> .npmrc - fi - - echo "Created the following .npmrc:"; cat .npmrc - - # Extract a few values from package.json - - NPM_PACKAGE_NAME=$(node -p "require('./package.json').name") - - NPM_PACKAGE_VERSION=$(node -p "require('./package.json').version") - - # Validate that the package name is properly scoped to the project's root namespace. - # For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention - - | - if [[ ! $NPM_PACKAGE_NAME =~ ^@$CI_PROJECT_ROOT_NAMESPACE/ ]]; then - echo "Invalid package scope! Packages must be scoped in the root namespace of the project, e.g. \"@${CI_PROJECT_ROOT_NAMESPACE}/${CI_PROJECT_NAME}\"" - echo 'For more information, see https://docs.gitlab.com/ee/user/packages/npm_registry/#package-naming-convention' - exit 1 - fi - - # Compare the version in package.json to all published versions. - # If the package.json version has not yet been published, run `npm publish`. - - | - if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then - npm publish - echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" - else - echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." - fi diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c4757edf74e..84eb860a168 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -14,6 +14,8 @@ module Gitlab UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds + LOAD_BALANCING_STICKING_NAMESPACE = 'ci/build/trace' + ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) LockedError = Class.new(StandardError) @@ -296,25 +298,31 @@ module Gitlab read_trace_artifact(job) { job.job_artifacts_trace } end - ## - # Overridden in EE - # - def destroy_stream(job) + def destroy_stream(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end - ## - # Overriden in EE - # - def read_trace_artifact(job) + def read_trace_artifact(build) + if consistent_archived_trace?(build) + ::Gitlab::Database::LoadBalancing::Sticking + .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + end + yield end + def consistent_archived_trace?(build) + ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false) + end + def being_watched_cache_key "gitlab:ci:trace:#{job.id}:watched" end end end end - -::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 7c2e39b1e53..9f24ba99201 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -229,13 +229,8 @@ module Gitlab def next_chunk @chunks_cache[chunk_index] = begin - if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) - ::Ci::BuildTraceChunk - .safe_find_or_create_by(build: build, chunk_index: chunk_index) - else - ::Ci::BuildTraceChunk - .new(build: build, chunk_index: chunk_index) - end + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) end end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index ce9efbda7ea..fcd70634630 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -11,7 +11,6 @@ module Gitlab :streamed, # new trace data has been sent by a runner :chunked, # new trace chunk has been created :mutated, # trace has been mutated when removing secrets - :overwrite, # runner requested overwritting a build trace :accepted, # scheduled chunks for migration and responded with 202 :finalized, # all live build trace chunks have been persisted :discarded, # failed to persist live chunks before timeout diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e2a8af9c26b..ef9ba1b73c7 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -24,6 +24,10 @@ module Gitlab self end + def compact + Collection.new(select { |variable| !variable.value.nil? }) + end + def concat(resources) return self if resources.nil? @@ -64,11 +68,19 @@ module Gitlab end def expand_value(value, keep_undefined: false) - value.gsub(ExpandVariables::VARIABLES_REGEXP) do + value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match - result = @variables_by_key[match[1] || match[2]]&.value - result ||= match[0] if keep_undefined - result + if match[:key] + # we matched variable + if variable = @variables_by_key[match[:key]] + variable.value + elsif keep_undefined + match[0] + end + else + # we escape sequence + match[0] + end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 77da2c4cb91..0217e6129ca 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -7,6 +7,9 @@ module Gitlab class Item include Gitlab::Utils::StrongMemoize + VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/.freeze.freeze + VARIABLE_REF_CHARS = %w[$ %].freeze + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? @@ -34,9 +37,9 @@ module Gitlab strong_memoize(:depends_on) do next if raw - next unless ExpandVariables.possible_var_reference?(value) + next unless self.class.possible_var_reference?(value) - value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + value.scan(VARIABLES_REGEXP).filter_map(&:last) end end @@ -64,6 +67,12 @@ module Gitlab end end + def self.possible_var_reference?(value) + return unless value + + VARIABLE_REF_CHARS.any? { |symbol| value.include?(symbol) } + end + def to_s return to_runner_variable.to_s unless depends_on diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f96a6629849..15cc0c28296 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -111,6 +111,22 @@ module Gitlab @ci_config.variables_with_data end + def yaml_variables_for(job_name) + job = jobs[job_name] + + return [] unless job + + Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: root_variables, + to: transform_to_yaml_variables(job[:job_variables]), + inheritance: job.fetch(:root_variables_inheritance, true) + ) + end + + def stage_for(job_name) + jobs.dig(job_name, :stage) + end + private def variables |