diff options
Diffstat (limited to 'lib/gitlab/ci')
36 files changed, 618 insertions, 47 deletions
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 071a8ef830f..8ed4dc61920 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -70,6 +70,10 @@ module Gitlab @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end + def included_templates + @context.expandset.filter_map { |i| i[:template] } + end + private def expand_config(config) @@ -98,7 +102,8 @@ module Gitlab project: project, sha: sha || project&.repository&.root_ref_sha, user: user, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project&.predefined_variables&.to_runner_variables) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 206dbaea272..6118ff49928 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -27,6 +27,7 @@ module Gitlab with_options allow_nil: true do validates :name, type: String + validates :public, boolean: true validates :untracked, boolean: true validates :paths, array_of_strings: true validates :paths, array_of_strings: { diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index cf6c2961ee7..e0adb1b19c2 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,14 +7,15 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user, :parent_pipeline + attr_reader :project, :sha, :user, :parent_pipeline, :variables attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline + @variables = variables @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index e74f5b33de7..fdb3e1b00f9 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -41,7 +41,8 @@ module Gitlab project: context.project, sha: context.sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index be479741784..114d493381c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -72,7 +72,8 @@ module Gitlab project: project, sha: sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 90692eafc3f..4d91cfd4c57 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -47,14 +48,14 @@ module Gitlab # convert location if String to canonical form def normalize_location(location) if location.is_a?(String) - normalize_location_string(location) + expanded_location = expand_variables(location) + normalize_location_string(expanded_location) else location.deep_symbolize_keys end end def expand_project_files(location) - return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true) return location unless location[:project] Array.wrap(location[:file]).map do |file| @@ -96,6 +97,33 @@ module Gitlab matching.first end + + def expand_variables(data) + return data unless ::Feature.enabled?(:variables_in_include_section_ci) + + if data.is_a?(String) + expand(data) + else + transform(data) + end + end + + def transform(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand(value.to_s) } + when String + expand(values) + else + values + end + end + end + + def expand(data) + ExpandVariables.expand(data, context.variables) + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index af1df933b36..7956cf14203 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -56,23 +56,19 @@ module Gitlab end def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: false) - end - - def self.seed_block_run_before_workflow_rules_enabled?(project) - ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true) + ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) end def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) + ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes) + ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) end def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false) + ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index fb795152abe..364e67db02b 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -18,9 +18,10 @@ module Gitlab end end - def initialize(project:, current_user:) + def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user + @sha = sha || project.repository.commit.sha end def validate(content, dry_run: false) @@ -51,7 +52,7 @@ module Gitlab content, project: @project, user: @current_user, - sha: @project.repository.commit.sha + sha: @sha ).execute Result.new( @@ -99,7 +100,8 @@ module Gitlab except: job[:except], environment: job[:environment], when: job[:when], - allow_failure: job[:allow_failure] + allow_failure: job[:allow_failure], + needs: job.dig(:needs_attributes) } end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 57f73c265b2..985639982aa 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type) - parsers.fetch(file_type.to_sym).new + def self.fabricate!(file_type, *args) + parsers.fetch(file_type.to_sym).new(*args) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 1edcbac2f25..eb3adf713d4 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -36,7 +36,7 @@ module Gitlab end def parse_node(key, value, coverage_report, context) - if key == 'sources' && value['source'].present? + if key == 'sources' && value && value['source'].present? parse_sources(value['source'], context) elsif key == 'package' Array.wrap(value).each do |item| diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 9662209f88e..f0548284001 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -5,6 +5,9 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + def perform! @pipeline.assign_attributes( source: @command.source, @@ -20,12 +23,34 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes) + locked: @command.project.latest_pipeline_locked, + variables_attributes: variables_attributes ) end def break? - false + @pipeline.errors.any? + end + + private + + def variables_attributes + variables = Array(@command.variables_attributes) + + # We allow parent pipelines to pass variables to child pipelines since + # these variables are coming from internal configurations. We will check + # permissions to :set_pipeline_variables when those are injected upstream, + # to the parent pipeline. + # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) + # the variables are provided from the outside and those should be guarded. + return variables if @command.creates_child_pipeline? + + if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) + error("Insufficient permissions to set pipeline variables") + variables = [] + end + + variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d05be54267c..815fe6bac6d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -79,6 +79,10 @@ module Gitlab bridge&.parent_pipeline end + def creates_child_pipeline? + bridge&.triggers_child_pipeline? + end + def metrics @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 083f0bec1df..7b537125b9b 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -19,13 +19,6 @@ module Gitlab # Build to prevent erroring out on ambiguous refs. pipeline.protected = @command.protected_ref? - unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - end - ## # Gather all runtime build/stage errors # diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb index f8e62949bea..67424635603 100644 --- a/lib/gitlab/ci/pipeline/chain/seed_block.rb +++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb @@ -9,8 +9,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## # Populate pipeline with block argument of CreatePipelineService#execute. # @@ -20,8 +18,6 @@ module Gitlab end def break? - return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - pipeline.errors.any? end end diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb new file mode 100644 index 00000000000..c1a7b4ed453 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class TemplateUsage < Chain::Base + def perform! + included_templates.each do |template| + track_event(template) + end + end + + def break? + false + end + + private + + def track_event(template) + Gitlab::UsageDataCounters::CiTemplateUniqueCounter + .track_unique_project_event(project_id: pipeline.project_id, template: template) + end + + def included_templates + command.yaml_processor_result.included_templates + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 8f1e690c081..e68d9020a21 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -19,7 +19,7 @@ module Gitlab end unless allowed_to_write_ref? - error("Insufficient permissions for protected ref '#{command.ref}'") + error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance. <a href=https://docs.gitlab.com/ee/ci/pipelines/#pipeline-security-on-protected-branches>Learn more</a>".html_safe) end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2271915a72b..fe3c2bca551 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -134,7 +134,7 @@ module Gitlab stage.seeds_names.include?(need[:name]) end - "#{name}: needs '#{need[:name]}'" unless result + "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result end.compact end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index beceac5423a..c024e794ad5 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -12,8 +12,6 @@ module Gitlab end def load! - return unless Feature.enabled?(:test_failure_history, project) - recent_failures_count.each do |key_hash, count| failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index ee785856fdd..37e2b7320e2 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -8,6 +8,10 @@ module Gitlab def self.common_helpers Status::Group::Common end + + def self.extended_statuses + [[Status::SuccessWarning]] + end end end end diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml new file mode 100644 index 00000000000..7182b96594d --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml @@ -0,0 +1,52 @@ +# +# You can use artifacts to pass data to jobs in later stages. +# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job might build an important file, and pass it to later jobs." + - echo "This is the content of the important file" > important-file.txt + artifacts: + paths: + - important-file.txt + +test-job-with-artifacts: + stage: test + script: + - echo "This job uses the artifact from the job in the earlier stage." + - cat important-file.txt + - echo "It creates another file, and adds it to the artifacts." + - echo "This is a second important file" > important-file2.txt + artifacts: + paths: + - important-file2.txt + +test-job-with-no-artifacts: + stage: test + dependencies: [] # Use to skip downloading any artifacts + script: + - echo "This job does not get the artifacts from other jobs." + - cat important-file.txt || exit 0 + +deploy-job-with-all-artifacts: + stage: deploy + script: + - echo "By default, jobs download all available artifacts." + - cat important-file.txt + - cat important-file2.txt + +deploy-job-with-1-artifact: + stage: deploy + dependencies: + - build-job # Download artifacts from only this job + script: + - echo "You can configure a job to download artifacts from only certain jobs." + - cat important-file.txt + - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml new file mode 100644 index 00000000000..382bac09ed7 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml @@ -0,0 +1,36 @@ +# +# You can define common tasks and run them before or after the main scripts in jobs. +# For more information, see: +# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script +# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script +# + +stages: + - test + +default: + before_script: + - echo "This script runs before the main script in every job, unless the job overrides it." + - echo "It may set up common dependencies, for example." + after_script: + - echo "This script runs after the main script in every job, unless the job overrides it." + - echo "It may do some common final clean up tasks" + +job-standard: + stage: test + script: + - echo "This job uses both of the globally defined before and after scripts." + +job-override-before: + stage: test + before_script: + - echo "Use a different before_script in this job." + script: + - echo "This job uses its own before_script, and the global after_script." + +job-override-after: + stage: test + after_script: + - echo "Use a different after_script in this job." + script: + - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml new file mode 100644 index 00000000000..5f27def74c9 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml @@ -0,0 +1,53 @@ +# +# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. +# To make a job manual, add when: manual to its configuration. +# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job is not a manual job" + +manual-build: + stage: build + script: + - echo "This manual job passes after you trigger it." + when: manual + +manual-build-allowed-to-fail: + stage: build + script: + - echo "This manual job fails after you trigger it." + - echo "It is allowed to fail, so the pipeline does not fail. + when: manual + allow_failure: true # Default behavior + +test-job: + stage: test + script: + - echo "This is a normal test job" + - echo "It runs when the when the build stage completes." + - echo "It does not need to wait for the manual jobs in the build stage to run." + +manual-test-not-allowed-to-fail: + stage: test + script: + - echo "This manual job fails after you trigger it." + - echo "It is NOT allowed to fail, so the pipeline is marked as failed + - echo "when this job completes." + - exit 1 + when: manual + allow_failure: false # Optional behavior + +deploy-job: + stage: deploy + script: + - echo "This is a normal deploy job" + - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, + - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml new file mode 100644 index 00000000000..aced628aacb --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml @@ -0,0 +1,33 @@ +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job runs in the build stage, which runs first." + +test-job1: + stage: test + script: + - echo "This job runs in the test stage." + - echo "It only starts when the job in the build stage completes successfully." + +test-job2: + stage: test + script: + - echo "This job also runs in the test stage." + - echo "This job can run at the same time as test-job2." + +deploy-job: + stage: deploy + script: + - echo "This job runs in the deploy stage." + - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml new file mode 100644 index 00000000000..2b8cf7bab44 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml @@ -0,0 +1,47 @@ +# +# Variables can be used to for more dynamic behavior in jobs and scripts. +# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html +# + +stages: + - test + +variables: + VAR1: "Variable 1 defined globally" + +use-a-variable: + stage: test + script: + - echo "You can use variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +override-a-variable: + stage: test + variables: + VAR1: "Variable 1 was overriden in in the job." + script: + - echo "You can override global variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +define-a-new-variable: + stage: test + variables: + VAR2: "Variable 2 is new and defined in the job only." + script: + - echo "You can mix global variables with variables defined in jobs." + - echo "The content of 'VAR1' is = $VAR1" + - echo "The content of 'VAR2' is = $VAR2" + +incorrect-variable-usage: + stage: test + script: + - echo "You can't use variables only defined in other jobs." + - echo "The content of 'VAR2' is = $VAR2" + +predefined-variables: + stage: test + script: + - echo "Some variables are predefined by GitLab CI/CD, for example:" + - echo "The commit author's username is $GITLAB_USER_LOGIN" + - echo "The commit branch is $CI_COMMIT_BRANCH" + - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml new file mode 100644 index 00000000000..c06ef83c180 --- /dev/null +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -0,0 +1,84 @@ +# This template is on early stage of development. +# Use it with caution. For usage instruction please read +# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md + +include: + # workflow rules to prevent duplicate detached pipelines + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + # auto devops build + - template: 'Jobs/Build.gitlab-ci.yml' + +stages: + - build + - test + - provision + - deploy + - destroy + +variables: + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_COMMIT_REF_SLUG} + TF_VAR_ENVIRONMENT_NAME: ${CI_PROJECT_PATH_SLUG}_${CI_PROJECT_ID}_${CI_COMMIT_REF_SLUG} + TF_VAR_SERVICE_DESK_EMAIL: incoming+${CI_PROJECT_PATH_SLUG}-${CI_PROJECT_ID}-issue-@incoming.gitlab.com + TF_VAR_SHORT_ENVIRONMENT_NAME: ${CI_PROJECT_ID}-${CI_COMMIT_REF_SLUG} + TF_VAR_SMTP_FROM: ${SMTP_FROM} + +cache: + paths: + - .terraform + +.needs_aws_vars: + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION' + when: on_success + - when: never + +terraform_apply: + stage: provision + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: terraform + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform init + - gitlab-terraform plan + - gitlab-terraform plan-json + - gitlab-terraform apply + +deploy: + stage: deploy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: deploy + before_script: + - cp /*.tf . + - cp /deploy.sh . + - cp /conf.nginx . + script: + - ./deploy.sh + artifacts: + reports: + dotenv: deploy.env + environment: + name: $CI_COMMIT_REF_SLUG + url: $DYNAMIC_ENVIRONMENT_URL + on_stop: terraform_destroy + +terraform_destroy: + variables: + GIT_STRATEGY: none + stage: destroy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform destroy -auto-approve + environment: + name: $CI_COMMIT_REF_SLUG + action: stop + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION && $CI_COMMIT_REF_PROTECTED == "false"' + when: manual + - when: never diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml new file mode 100644 index 00000000000..504ece611ca --- /dev/null +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -0,0 +1,29 @@ +code_quality: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate dart_code_metrics + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - metrics lib -r codeclimate > gl-code-quality-report.json + artifacts: + reports: + codequality: gl-code-quality-report.json + +test: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate junitreport + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - flutter test --machine --coverage | tojunit -o report.xml + - lcov --summary coverage/lcov.info + - genhtml coverage/lcov.info --output=coverage + coverage: '/lines\.*: \d+\.\d+\%/' + artifacts: + name: coverage + paths: + - $CI_PROJECT_DIR/coverage + reports: + junit: report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 2ae9730ec1a..501d8737acd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 23dfeda31cc..192b1509fdc 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.36.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml new file mode 100644 index 00000000000..fc1acd09714 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -0,0 +1,43 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + +dast: + stage: dast + image: + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - if [ -z "$DAST_WEBSITE$DAST_API_SPECIFICATION" ]; then echo "Either DAST_WEBSITE or DAST_API_SPECIFICATION must be set. See https://docs.gitlab.com/ee/user/application_security/dast/#configuration for more details." && exit 1; fi + - /analyze + artifacts: + reports: + dast: gl-dast-report.json + rules: + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && $DAST_WEBSITE == null && + $DAST_API_SPECIFICATION == null + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && + $DAST_WEBSITE + - if: $CI_COMMIT_BRANCH && + $DAST_API_SPECIFICATION diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index f4ee8ebd47e..56c6fbd96bc 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -10,6 +10,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SCAN_KUBERNETES_MANIFESTS: "false" @@ -44,6 +45,8 @@ bandit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /bandit/ exists: @@ -58,6 +61,8 @@ brakeman-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: @@ -72,6 +77,8 @@ eslint-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /eslint/ exists: @@ -90,6 +97,8 @@ flawfinder-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ exists: @@ -105,6 +114,8 @@ kubesec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -118,6 +129,8 @@ gosec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /gosec/ exists: @@ -136,6 +149,8 @@ mobsf-android-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -155,6 +170,8 @@ mobsf-ios-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -170,6 +187,8 @@ nodejs-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: @@ -184,6 +203,8 @@ phpcs-security-audit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ exists: @@ -198,6 +219,8 @@ pmd-apex-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: @@ -212,6 +235,8 @@ security-code-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ exists: @@ -227,6 +252,8 @@ sobelow-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: @@ -239,6 +266,8 @@ spotbugs-sast: variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 8ca1d2e08ba..d2a6fa06dd8 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -37,6 +37,7 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 377c72e8031..7e2828d010f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -17,6 +17,7 @@ variables: cache: paths: - .terraform + - .terraform.lock.hcl before_script: - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 910e711f046..c2db0fc44f1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -19,6 +19,7 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ + - ${TF_ROOT}/.terraform.lock.hcl .init: &init stage: init diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb new file mode 100644 index 00000000000..6abc6a5644f --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sorted + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(variables) + @variables = variables + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + return if Feature.disabled?(:variable_inside_variable) + + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + errors = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{errors.join(', ')}" if errors.any? + end + end + + # sort sorts an array of variables, ignoring unknown variable references. + # If a circular variable reference is found, the original array is returned + def sort + return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if errors + + tsort + end + + private + + def tsort_each_node(&block) + @variables.each(&block) + end + + def tsort_each_child(variable, &block) + each_variable_reference(variable[:value], &block) + end + + def input_vars + strong_memoize(:input_vars) do + @variables.index_by { |env| env.fetch(:key) } + end + end + + def walk_references(value) + return unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| + yield(input_vars, var_ref.first) + end + end + + def each_variable_reference(value) + walk_references(value) do |vars_hash, ref_var_name| + variable = vars_hash.dig(ref_var_name) + yield variable if variable + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index ee55eb8b22a..dc4951f76bb 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -10,12 +10,6 @@ module Gitlab class YamlProcessor ValidationError = Class.new(StandardError) - def self.validation_message(content, opts = {}) - result = new(content, opts).execute - - result.errors.first - end - def initialize(config_content, opts = {}) @config_content = config_content @opts = opts diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index cd7d781a574..86749cda9c7 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -53,6 +53,10 @@ module Gitlab @stages ||= @ci_config.stages end + def included_templates + @included_templates ||= @ci_config.included_templates + end + def build_attributes(name) job = jobs.fetch(name.to_sym, {}) |