diff options
Diffstat (limited to 'lib/gitlab/ci')
25 files changed, 332 insertions, 84 deletions
diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index afe0ccb361e..7ade9ca5085 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -35,7 +35,10 @@ module Gitlab # patterns can be matched only when branch or tag is used # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? - if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern, fallback: true) + regexp = Gitlab::UntrustedRegexp::RubySyntax + .fabricate(pattern, fallback: true, project: pipeline.project) + + if regexp regexp.match?(pipeline.ref) else pattern == pipeline.ref diff --git a/lib/gitlab/ci/build/status/reason.rb b/lib/gitlab/ci/build/status/reason.rb new file mode 100644 index 00000000000..82e07faef63 --- /dev/null +++ b/lib/gitlab/ci/build/status/reason.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Status + class Reason + attr_reader :build, :failure_reason, :exit_code + + def initialize(build, failure_reason, exit_code = nil) + @build = build + @failure_reason = failure_reason + @exit_code = exit_code + end + + def failure_reason_enum + ::CommitStatus.failure_reasons[failure_reason] + end + + def force_allow_failure? + return false if exit_code.nil? + + !build.allow_failure? && build.allowed_to_fail_with_code?(exit_code) + end + + def self.fabricate(build, reason) + if reason.is_a?(self) + new(build, reason.failure_reason, reason.exit_code) + else + new(build, reason) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 42b487fdf81..4c98941e032 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -36,7 +36,7 @@ module Gitlab end @root = self.logger.instrument(:config_compose) do - Entry::Root.new(@config).tap(&:compose!) + Entry::Root.new(@config, project: project, user: user).tap(&:compose!) end rescue *rescue_errors => e raise Config::ConfigError, e.message diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index e6290ef2479..41a3c87037b 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -59,7 +59,8 @@ module Gitlab entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.', - reserved: true + reserved: true, + deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0', documentation: 'https://docs.gitlab.com/ee/ci/yaml/#deprecated-keywords' } entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', @@ -122,8 +123,9 @@ module Gitlab # Deprecated `:types` key workaround - if types are defined and # stages are not defined we use types definition as stages. # - if types_defined? && !stages_defined? - @entries[:stages] = @entries[:types] + if types_defined? + @entries[:stages] = @entries[:types] unless stages_defined? + log_and_warn_deprecated_entry(@entries[:types]) end @entries.delete(:types) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb new file mode 100644 index 00000000000..278353220e4 --- /dev/null +++ b/lib/gitlab/ci/jwt_v2.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class JwtV2 < Jwt + private + + def reserved_claims + super.merge( + iss: Settings.gitlab.base_url, + sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", + aud: Settings.gitlab.base_url + ) + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 15b0ff3c04d..54b54bd0514 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,13 +9,13 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument(:pipeline_save) do + logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - tags = extract_tag_list_by_status - - pipeline.transaction do - pipeline.save! - CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + with_bulk_insert_tags do + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + end end end end @@ -29,32 +29,26 @@ module Gitlab private - def statuses - strong_memoize(:statuses) do - pipeline.stages.flat_map(&:statuses) + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) end end - # We call `job.tag_list=` to assign tags to the jobs from the - # Chain::Seed step which uses the `@tag_list` instance variable to - # store them on the record. We remove them here because we want to - # bulk insert them, otherwise they would be inserted and assigned one - # by one with callbacks. We must use `remove_instance_variable` - # because having the instance variable defined would still run the callbacks - def extract_tag_list_by_status - return {} unless bulk_insert_tags? - - statuses.each.with_object({}) do |job, acc| - tag_list = job.clear_memoization(:tag_list) - next unless tag_list - - acc[job.name] = tag_list - end + def with_bulk_insert_tags + previous = Thread.current['ci_bulk_insert_tags'] + Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + yield + ensure + Thread.current['ci_bulk_insert_tags'] = previous end - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + def statuses + strong_memoize(:statuses) do + pipeline + .stages + .flat_map(&:statuses) + .select { |status| status.respond_to?(:tag_list) } end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index b92aa89d62d..b913ba3c87d 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -5,8 +5,6 @@ module Gitlab module Pipeline module Chain class CreateDeployments < Chain::Base - DeploymentCreationError = Class.new(StandardError) - def perform! return unless pipeline.create_deployment_in_separate_transaction? @@ -24,18 +22,7 @@ module Gitlab end def create_deployment(build) - return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? - - deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment - .new(build, build.persisted_environment).to_resource - - return unless deployment - - deployment.deployable = build - deployment.save! - rescue ActiveRecord::RecordInvalid => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception( - DeploymentCreationError.new(e.message), build_id: build.id) + ::Deployments::CreateForBuildService.new.execute(build) end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 356eeb76908..feae123f216 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -53,13 +53,18 @@ module Gitlab end def context - Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + Gitlab::Ci::Pipeline::Seed::Context.new( + pipeline, + root_variables: root_variables, + logger: logger + ) end def root_variables logger.instrument(:pipeline_seed_merge_variables) do ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + @command.yaml_processor_result.root_variables, + @command.workflow_rules_result.variables ) end end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 97f7dddd09a..fbba12c11a9 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -37,6 +37,16 @@ module Gitlab result end + def instrument_with_sql(operation, &block) + op_start_db_counters = current_db_counter_payload + + result = instrument(operation, &block) + + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + + result + end + def observe(operation, value) return unless enabled? @@ -50,11 +60,20 @@ module Gitlab class: self.class.name.to_s, pipeline_creation_caller: caller, project_id: project.id, - pipeline_id: pipeline.id, pipeline_persisted: pipeline.persisted?, pipeline_source: pipeline.source, pipeline_creation_service_duration_s: age - }.stringify_keys.merge(observations_hash) + } + + if pipeline.persisted? + attributes[:pipeline_builds_tags_count] = pipeline.tags_count + attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count + attributes[:pipeline_id] = pipeline.id + end + + attributes.compact! + attributes.stringify_keys! + attributes.merge!(observations_hash) destination.info(attributes) end @@ -97,6 +116,19 @@ module Gitlab def observations @observations ||= Hash.new { |hash, key| hash[key] = [] } end + + def observe_sql_counters(operation, start_db_counters, end_db_counters) + end_db_counters.each do |key, value| + result = value - start_db_counters.fetch(key, 0) + next if result == 0 + + observe("#{operation}_#{key}", result) + end + end + + def current_db_counter_payload + ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 762292f0fa3..5a0ad695741 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -41,12 +41,14 @@ module Gitlab def included? strong_memoize(:inclusion) do - if @using_rules - rules_result.pass? - elsif @using_only || @using_except - all_of_only? && none_of_except? - else - true + logger.instrument(:pipeline_seed_build_inclusion) do + if @using_rules + rules_result.pass? + elsif @using_only || @using_except + all_of_only? && none_of_except? + else + true + end end end end @@ -122,6 +124,8 @@ module Gitlab private + delegate :logger, to: :@context + def all_of_only? @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) } end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb index 6194a78f682..c0b8ebeb833 100644 --- a/lib/gitlab/ci/pipeline/seed/context.rb +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -5,11 +5,18 @@ module Gitlab module Pipeline module Seed class Context - attr_reader :pipeline, :root_variables + attr_reader :pipeline, :root_variables, :logger - def initialize(pipeline, root_variables: []) + def initialize(pipeline, root_variables: [], logger: nil) @pipeline = pipeline @root_variables = root_variables + @logger = logger || build_logger + end + + private + + def build_logger + ::Gitlab::Ci::Pipeline::Logger.new(project: pipeline.project) end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7f45d626922..54fb1d19ea8 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -69,17 +69,6 @@ module Gitlab self.class.attempt_counter.increment end - # rubocop: disable CodeReuse/ActiveRecord - def jobs_running_for_project(job) - return '+Inf' unless runner.instance_type? - - # excluding currently started job - running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 - running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" - end - # rubocop: enable CodeReuse/ActiveRecord - def increment_queue_operation(operation) self.class.increment_queue_operation(operation) end @@ -242,6 +231,32 @@ module Gitlab Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = running_jobs_relation(job) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + + if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + running_jobs_count + else + "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + end + + def running_jobs_relation(job) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) + else + job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 7e5afbad806..a4434e2c144 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -14,7 +14,8 @@ module Gitlab Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, - Status::Build::Skipped], + Status::Build::Skipped, + Status::Build::WaitingForApproval], [Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::FailedUnmetPrerequisites, diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb new file mode 100644 index 00000000000..59869a947a9 --- /dev/null +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Build + class WaitingForApproval < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: 'Waiting for approval', + content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + } + end + + def self.matches?(build, user) + build.waiting_for_deployment_approval? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index a299df7e2d9..29f3731a9b4 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -4,12 +4,13 @@ module Gitlab module Ci module Tags class BulkInsert + include Gitlab::Utils::StrongMemoize + TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses, tag_list_by_status) + def initialize(statuses) @statuses = statuses - @tag_list_by_status = tag_list_by_status end def insert! @@ -20,7 +21,18 @@ module Gitlab private - attr_reader :statuses, :tag_list_by_status + attr_reader :statuses + + def tag_list_by_status + strong_memoize(:tag_list_by_status) do + statuses.each.with_object({}) do |status, acc| + tag_list = status.tag_list + next unless tag_list + + acc[status] = tag_list + end + end + end def persist_build_tags! all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) @@ -54,7 +66,7 @@ module Gitlab def build_taggings_attributes(tag_records_by_name) taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status.name] + tag_list = tag_list_by_status[status] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 00b771f1e5c..6942631a97f 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 18f0f20203d..42487cc0c67 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,8 @@ variables: image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] allow_failure: true + variables: + GIT_DEPTH: "50" # `rules` must be overridden explicitly by each child job # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: @@ -29,8 +31,16 @@ secret_detection: script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi - - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt + - | + git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt + if [[ $(wc -l <${CI_COMMIT_SHA}_commit_list.txt) -eq "0" ]]; then + # if git log produces 0 or 1 commits we should scan $CI_COMMIT_SHA only + export SECRET_DETECTION_COMMITS=$CI_COMMIT_SHA + else + # +1 because busybox wc only countsĀ \n and there is no trailing \n + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt + fi - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 0c8b98dc1cf..1660a9250e3 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,7 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 7243f240eed..f7f016b5e57 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -33,6 +33,7 @@ coverage_fuzzing_unlicensed: before_script: - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN + - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64 diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml new file mode 100644 index 00000000000..6888e955467 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -0,0 +1,27 @@ +stages: + - build + - test + - deploy + - dast + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast: + stage: dast + image: $DAST_API_IMAGE + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index e554742735c..12c987a8d37 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -5,9 +5,11 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml stages: - validate + - test - build - deploy diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index d57f3888ec0..7f43d91e6d7 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -26,7 +26,6 @@ module Gitlab delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum - return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml) return unless object_store_config.enabled? return if trace_artifact.local_store? diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 2d31049a0c9..dd435ba05b7 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -11,10 +11,6 @@ module Gitlab delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true - delegate :valid?, to: :stream, allow_nil: true - - alias_method :present?, :valid? - def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode @@ -24,6 +20,7 @@ module Gitlab def valid? self.stream.present? end + alias_method :present?, :valid? def file? self.path.present? diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 3e2c2c7fc1a..4c777527ebc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -13,12 +13,76 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables(job)) + + next variables unless pipeline.use_variables_builder_definitions? + + variables.concat(project.predefined_variables) + variables.concat(pipeline.predefined_variables) + variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner + variables.concat(kubernetes_variables(job)) + variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(job.yaml_variables) + variables.concat(user_variables(job.user)) + variables.concat(job.dependency_variables) if dependencies + variables.concat(secret_instance_variables(ref: job.git_ref)) + variables.concat(secret_group_variables(environment: environment, ref: job.git_ref)) + variables.concat(secret_project_variables(environment: environment, ref: job.git_ref)) + variables.concat(job.trigger_request.user_variables) if job.trigger_request + variables.concat(pipeline.variables) + variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + end + end + + def kubernetes_variables(job) + ::Gitlab::Ci::Variables::Collection.new.tap do |collection| + # Should get merged with the cluster kubeconfig in deployment_variables, see + # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + template = ::Ci::GenerateKubeconfigService.new(job).execute + + if template.valid? + collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) + end end end + def deployment_variables(environment:, job:) + return [] unless environment + + project.deployment_variables( + environment: environment, + kubernetes_namespace: job.expanded_kubernetes_namespace + ) + end + + def user_variables(user) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables if user.blank? + + variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) + variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) + variables.append(key: 'GITLAB_USER_LOGIN', value: user.username) + variables.append(key: 'GITLAB_USER_NAME', value: user.name) + end + end + + def secret_instance_variables(ref:) + project.ci_instance_variables_for(ref: ref) + end + + def secret_group_variables(environment:, ref:) + return [] unless project.group + + project.group.ci_variables_for(ref, project, environment: environment) + end + + def secret_project_variables(environment:, ref:) + project.ci_variables_for(ref: ref, environment: environment) + end + private attr_reader :pipeline + delegate :project, to: :pipeline def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 296b0cfded2..553508c8638 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -86,11 +86,19 @@ module Gitlab def validate_job_needs!(name, job) return unless needs = job.dig(:needs, :job) + validate_duplicate_needs!(name, needs) + needs.each do |need| validate_job_dependency!(name, need[:name], 'need') end end + def validate_duplicate_needs!(name, needs) + unless needs.uniq == needs + error!("#{name} has duplicate entries in the needs section.") + end + end + def validate_job_dependency!(name, dependency, dependency_type = 'dependency') unless @jobs[dependency.to_sym] error!("#{name} job: undefined #{dependency_type}: #{dependency}") |