diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 13:18:24 +0000 |
commit | 0653e08efd039a5905f3fa4f6e9cef9f5d2f799c (patch) | |
tree | 4dcc884cf6d81db44adae4aa99f8ec1233a41f55 /lib/gitlab/ci | |
parent | 744144d28e3e7fddc117924fef88de5d9674fe4c (diff) | |
download | gitlab-ce-0653e08efd039a5905f3fa4f6e9cef9f5d2f799c.tar.gz |
Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
39 files changed, 527 insertions, 95 deletions
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index d576953c1a0..3cfed8e5e2c 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,7 +45,7 @@ module Gitlab end def read_zip_file!(file_path) - if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: false) + if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: :yaml) read_with_new_artifact_file_reader(file_path) else read_with_legacy_artifact_file_reader(file_path) diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index eaaf9f69102..145772c7a92 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -53,7 +53,7 @@ module Gitlab description: 'Set retry default value.', inherit: false - entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + entry :tags, Entry::Tags, description: 'Set the default tags.', inherit: false diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index bd4d5f33689..f867189d521 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -85,7 +85,7 @@ module Gitlab description: 'Retry configuration for this job.', inherit: true - entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + entry :tags, Entry::Tags, description: 'Set the tags.', inherit: true diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 3543b5493bd..2549c35ebd6 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -16,6 +16,7 @@ module Gitlab PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit allow_failure when needs resource_group].freeze + MAX_NESTING_LEVEL = 10 included do validations do @@ -31,7 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true - validates :rules, nested_array_of_hashes: true + validates :rules, nested_array_of_hashes_or_arrays: { max_level: MAX_NESTING_LEVEL } validates :resource_group, type: String end end diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb new file mode 100644 index 00000000000..ca3b48372e2 --- /dev/null +++ b/lib/gitlab/ci/config/entry/tags.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an array of tags. + # + class Tags < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + TAGS_LIMIT = 50 + + validations do + validates :config, array_of_strings: true + + validate do + next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml) + + if config.is_a?(Array) && config.size >= TAGS_LIMIT + errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT }) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index bc03658aab8..7334a112ccf 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -6,8 +6,40 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' VALID_SYNTAX_SAMPLE_CRON = '* * * * *' - def self.parse_natural(expression, cron_timezone = 'UTC') - new(Fugit::Nat.parse(expression)&.original, cron_timezone) + class << self + def parse_natural(expression, cron_timezone = 'UTC') + new(Fugit::Nat.parse(expression)&.original, cron_timezone) + end + + # This method generates compatible expressions that can be + # parsed by Fugit::Nat.parse to generate a cron line. + # It takes start date of the cron and cadence in the following format: + # cadence = { + # unit: 'day/week/month/year' + # duration: 1 + # } + def parse_natural_with_timestamp(starts_at, cadence) + case cadence[:unit] + when 'day' # Currently supports only 'every 1 day'. + "#{starts_at.min} #{starts_at.hour} * * *" + when 'week' # Currently supports only 'every 1 week'. + "#{starts_at.min} #{starts_at.hour} * * #{starts_at.wday}" + when 'month' + unless [1, 3, 6, 12].include?(cadence[:duration]) + raise NotImplementedError, "The cadence #{cadence} is not supported" + end + + "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{fall_in_months(cadence[:duration], starts_at)} *" + when 'year' # Currently supports only 'every 1 year'. + "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{starts_at.month} *" + else + raise NotImplementedError, "The cadence unit #{cadence[:unit]} is not implemented" + end + end + + def fall_in_months(offset, start_date) + (1..(12 / offset)).map { |i| start_date.next_month(offset * i).month }.join(',') + end end def initialize(cron, cron_timezone = 'UTC') diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index cd2c135dd7e..8c1067b9bc6 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -21,7 +21,7 @@ module Gitlab def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user - @sha = sha || project.repository.commit&.sha + @sha = sha || project&.repository&.commit&.sha end def validate(content, dry_run: false) diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 41acb4d5040..1cf4f252ab9 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -86,6 +86,7 @@ module Gitlab def create_finding(data, remediations = []) identifiers = create_identifiers(data['identifiers']) + flags = create_flags(data['flags']) links = create_links(data['links']) location = create_location(data['location'] || {}) signatures = create_signatures(tracking_data(data)) @@ -111,6 +112,7 @@ module Gitlab scanner: create_scanner(data['scanner']), scan: report&.scan, identifiers: identifiers, + flags: flags, links: links, remediations: remediations, raw_metadata: data.to_json, @@ -205,6 +207,18 @@ module Gitlab url: identifier['url'])) end + def create_flags(flags) + return [] unless flags.is_a?(Array) + + flags.map { |flag| create_flag(flag) }.compact + end + + def create_flag(flag) + return unless flag.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Flag.new(type: flag['type'], origin: flag['origin'], description: flag['description']) + end + def create_links(links) return [] unless links.is_a?(Array) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 3d92886cba8..143b930c669 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -12,7 +12,7 @@ module Gitlab end def initialize(report_type) - @report_type = report_type + @report_type = report_type.to_sym end delegate :validate, to: :schemer diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index d3bc3a38f1f..6feb693221b 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -5,9 +5,6 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base - include Gitlab::Allowable - include Chain::Helpers - def perform! @pipeline.assign_attributes( source: @command.source, @@ -23,35 +20,12 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - locked: @command.project.default_pipeline_lock, - variables_attributes: variables_attributes - ) + locked: @command.project.default_pipeline_lock) end def break? @pipeline.errors.any? end - - private - - def variables_attributes - variables = Array(@command.variables_attributes) - - # We allow parent pipelines to pass variables to child pipelines since - # these variables are coming from internal configurations. We will check - # permissions to :set_pipeline_variables when those are injected upstream, - # to the parent pipeline. - # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) - # the variables are provided from the outside and those should be guarded. - return variables if @command.creates_child_pipeline? - - if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) - error("Insufficient permissions to set pipeline variables") - variables = [] - end - - variables - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb index eb49c56bcd7..b5d63691849 100644 --- a/lib/gitlab/ci/pipeline/chain/build/associations.rb +++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb @@ -6,7 +6,25 @@ module Gitlab module Chain class Build class Associations < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + def perform! + assign_pipeline_variables + assign_source_pipeline + end + + def break? + @pipeline.errors.any? + end + + private + + def assign_pipeline_variables + @pipeline.variables_attributes = variables_attributes + end + + def assign_source_pipeline return unless @command.bridge @pipeline.build_source_pipeline( @@ -17,8 +35,45 @@ module Gitlab ) end - def break? - false + def variables_attributes + variables = Array(@command.variables_attributes) + variables = apply_permissions(variables) + validate_uniqueness(variables) + end + + def apply_permissions(variables) + # We allow parent pipelines to pass variables to child pipelines since + # these variables are coming from internal configurations. We will check + # permissions to :set_pipeline_variables when those are injected upstream, + # to the parent pipeline. + # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) + # the variables are provided from the outside and those should be guarded. + return variables if @command.creates_child_pipeline? + + if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) + error("Insufficient permissions to set pipeline variables") + variables = [] + end + + variables + end + + def validate_uniqueness(variables) + duplicated_keys = variables + .map { |var| var[:key] } + .tally + .filter_map { |key, count| key if count > 1 } + + if duplicated_keys.empty? + variables + else + error(duplicate_variables_message(duplicated_keys), config_error: true) + [] + end + end + + def duplicate_variables_message(keys) + "Duplicate variable #{'name'.pluralize(keys.size)}: #{keys.join(', ')}" end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 1c0dfbdbee3..f637001f9f8 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -7,15 +7,19 @@ module Gitlab class CancelPendingPipelines < Chain::Base include Chain::Helpers + BATCH_SIZE = 25 + + # rubocop: disable CodeReuse/ActiveRecord def perform! return unless project.auto_cancel_pending_pipelines? Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| - cancelables.find_each do |cancelable| - cancelable.auto_cancel_running(pipeline) + cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| + auto_cancel_interruptible_pipelines(cancelables_batch.ids) end end end + # rubocop: enable CodeReuse/ActiveRecord def break? false @@ -23,16 +27,21 @@ module Gitlab private - # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - project.all_pipelines.ci_and_parent_sources - .where(ref: pipeline.ref) - .where.not(id: pipeline.same_family_pipeline_ids) - .where.not(sha: project.commit(pipeline.ref).try(:id)) + project.all_pipelines.created_after(1.week.ago) + .ci_and_parent_sources + .for_ref(pipeline.ref) + .id_not_in(pipeline.same_family_pipeline_ids) + .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled + end + + def auto_cancel_interruptible_pipelines(pipeline_ids) + ::Ci::Pipeline + .id_in(pipeline_ids) .with_only_interruptible_builds + .each { |cancelable| cancelable.auto_cancel_running(pipeline) } end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 626eba97817..c9bc4ec411d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -87,6 +87,13 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def observe_step_duration(step_class, duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) + metrics.pipeline_creation_step_duration_histogram + .observe({ step: step_class.name }, duration.seconds) + end + end + def observe_creation_duration(duration) metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb index 8a19e433483..092e7d43371 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb @@ -11,8 +11,12 @@ module Gitlab strong_memoize(:content) do next unless external_project_path? - path_file, path_project = ci_config_path.split('@', 2) - YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }]) + path_file, path_project, ref = extract_location_tokens + + config_location = { 'project' => path_project, 'file' => path_file } + config_location['ref'] = ref if ref.present? + + YAML.dump('include' => [config_location]) end end @@ -26,6 +30,18 @@ module Gitlab def external_project_path? ci_config_path =~ /\A.+(yml|yaml)@.+\z/ end + + # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname + def extract_location_tokens + path_file, path_project = ci_config_path.split('@', 2) + + if path_project.include? ":" + project, ref = path_project.split(':', 2) + [path_file, project, ref] + else + [path_file, path_project] + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index bbfc6759b35..845eb6c7a42 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -14,9 +14,16 @@ module Gitlab def build! @sequence.each do |step_class| + step_start = ::Gitlab::Metrics::System.monotonic_time step = step_class.new(@pipeline, @command) step.perform! + + @command.observe_step_duration( + step_class, + ::Gitlab::Metrics::System.monotonic_time - step_start + ) + break if step.break? end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 10de77afe74..28df9f5386c 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -4,6 +4,8 @@ module Gitlab module Ci module Pipeline class Metrics + extend Gitlab::Utils::StrongMemoize + def self.pipeline_creation_duration_histogram name = :gitlab_ci_pipeline_creation_duration_seconds comment = 'Pipeline creation duration' @@ -13,6 +15,17 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_creation_step_duration_histogram + strong_memoize(:pipeline_creation_step_histogram) do + name = :gitlab_ci_pipeline_creation_step_duration_seconds + comment = 'Duration of each pipeline creation step' + labels = { step: nil } + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 15.0, 20.0, 50.0, 240.0] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + def self.pipeline_security_orchestration_policy_processing_duration_histogram name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds comment = 'Pipeline security orchestration policy processing duration' diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index c393fed26de..934bf22d8ad 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -15,12 +15,7 @@ module Gitlab @context = context @pipeline = context.pipeline @seed_attributes = attributes - @stages_for_needs_lookup = if Feature.enabled?(:ci_same_stage_job_needs, @pipeline.project, default_enabled: :yaml) - (previous_stages + [current_stage]).compact - else - previous_stages - end - + @stages_for_needs_lookup = (previous_stages + [current_stage]).compact @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @@ -123,6 +118,8 @@ module Gitlab return { environment: nil } end + build.persisted_environment = environment + { deployment: Seed::Deployment.new(build, environment).to_resource, metadata_attributes: { diff --git a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index f8ea6d4184c..a29fef6eb34 100644 --- a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -28,7 +28,16 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { variables }) + end + end + + def variables + processable.simple_variables.tap do |variables| + # Adding persisted environment variables + if processable.persisted_environment.present? + variables.concat(processable.persisted_environment.predefined_variables) + end end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 859aeb35f26..7f45d626922 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -97,7 +97,9 @@ module Gitlab def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) - self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f) + size = size_proc.call.to_f + self.class.queue_size_total.observe({ runner_type: runner_type }, size) + self.class.current_queue_size.set({ runner_type: runner_type }, size) end def observe_queue_time(metric, runner_type) @@ -199,6 +201,15 @@ module Gitlab end end + def self.current_queue_size + strong_memoize(:current_queue_size) do + name = :gitlab_ci_current_queue_size + comment = 'Current size of initialized CI/CD builds queue' + + Gitlab::Metrics.gauge(name, comment) + end + end + def self.queue_iteration_duration_seconds strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index dc1c51b3ed0..39531e12f69 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -10,6 +10,7 @@ module Gitlab attr_reader :compare_key attr_reader :confidence attr_reader :identifiers + attr_reader :flags attr_reader :links attr_reader :location attr_reader :metadata_version @@ -30,10 +31,11 @@ module Gitlab delegate :file_path, :start_line, :end_line, to: :location - def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers + @flags = flags @links = links @location = location @metadata_version = metadata_version @@ -58,6 +60,7 @@ module Gitlab compare_key confidence identifiers + flags links location metadata_version diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb new file mode 100644 index 00000000000..7e6cc758864 --- /dev/null +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Flag + attr_reader :type, :origin, :description + + MAP = { 'flagged-as-likely-false-positive' => :false_positive }.freeze + DEFAULT_FLAG_TYPE = :false_positive + + def flag_type + MAP.fetch(type, DEFAULT_FLAG_TYPE) + end + + def initialize(type: nil, origin: nil, description: nil) + @type = type + @origin = origin + @description = description + end + + def to_hash + { + flag_type: flag_type, + origin: origin, + description: description + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index dbbb9a01dab..ee210e51232 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -32,7 +32,8 @@ module Gitlab user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', no_matching_runner: 'no matching runner available', - trace_size_exceeded: 'log size limit exceeded' + trace_size_exceeded: 'log size limit exceeded', + builds_disabled: 'project builds are disabled' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml new file mode 100644 index 00000000000..9f0e9bcc1f2 --- /dev/null +++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml @@ -0,0 +1,87 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml + +# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny +# If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. + +image: openjdk:11-jdk + +variables: + + # ANDROID_COMPILE_SDK is the version of Android you're compiling with. + # It should match compileSdkVersion. + ANDROID_COMPILE_SDK: "30" + + # ANDROID_BUILD_TOOLS is the version of the Android build tools you are using. + # It should match buildToolsVersion. + ANDROID_BUILD_TOOLS: "30.0.3" + + # It's what version of the command line tools we're going to download from the official site. + # Official Site-> https://developer.android.com/studio/index.html + # There, look down below at the cli tools only, sdk tools package is of format: + # commandlinetools-os_type-ANDROID_SDK_TOOLS_latest.zip + # when the script was last modified for latest compileSdkVersion, it was which is written down below + ANDROID_SDK_TOOLS: "7583922" + +# Packages installation before running script +before_script: + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + + # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it + - export ANDROID_SDK_ROOT="${PWD}/android-home" + # Create a new directory at specified location + - install -d $ANDROID_SDK_ROOT + # Here we are installing androidSDK tools from official source, + # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) + # after that unzipping those tools and + # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build + - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + # move to the archive at ANDROID_SDK_ROOT + - pushd $ANDROID_SDK_ROOT + - unzip -d cmdline-tools cmdline-tools.zip + - pushd cmdline-tools + # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary + - mv cmdline-tools tools || true + - popd + - popd + - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/ + + # Nothing fancy here, just checking sdkManager version + - sdkmanager --version + + # use yes to accept all licenses + - yes | sdkmanager --licenses || true + - sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" + - sdkmanager "platform-tools" + - sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" + + # Not necessary, but just for surity + - chmod +x ./gradlew + +# Basic android and gradle stuff +# Check linting +lintDebug: + interruptible: true + stage: build + script: + - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + +# Make Project +assembleDebug: + interruptible: true + stage: build + script: + - ./gradlew assembleDebug + artifacts: + paths: + - app/build/outputs/ + +# Run all tests, if any fails, interrupt the pipeline(fail it) +debugTests: + interruptible: true + stage: test + script: + - ./gradlew -Pci --console=plain :app:testDebug diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index cf99d722e4d..5efa557d7eb 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,6 +1,9 @@ +variables: + AUTO_BUILD_IMAGE_VERSION: 'v1.0.0' + build: stage: build - image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0' + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' variables: DOCKER_TLS_CERTDIR: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml new file mode 100644 index 00000000000..6a3b0cfa9e7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -0,0 +1,41 @@ +# WARNING: This latest template is for internal FEATURE-FLAG TESTING ONLY. +# It is not meant to be used with `include:`. +# This template is scheduled for removal when testing is complete: https://gitlab.com/gitlab-org/gitlab/-/issues/337987 + +variables: + AUTO_BUILD_IMAGE_VERSION: 'v1.3.1' + +build: + stage: build + image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}' + variables: + DOCKER_TLS_CERTDIR: '' + services: + - name: 'docker:20.10.6-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] + script: + - | + if [[ -z "$CI_COMMIT_TAG" ]]; then + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} + else + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} + fi + - /build/build.sh + rules: + - if: '$BUILD_DISABLED' + when: never + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +build_artifact: + stage: build + script: + - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" + - exit 1 + rules: + - if: '$BUILD_DISABLED' + when: never + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 43ecc4b96d5..00b771f1e5c 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 208951fa1a1..e0627b85aba 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ +variables: + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 5c466f0984c..2df985cfbb5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,8 @@ +variables: + AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" dependencies: [] review: @@ -96,6 +99,8 @@ canary: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' @@ -125,6 +130,8 @@ canary: production: <<: *production_template rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$STAGING_ENABLED' @@ -141,6 +148,8 @@ production_manual: <<: *production_template allow_failure: false rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_ENABLED' @@ -177,6 +186,8 @@ production_manual: resource_group: production allow_failure: true rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' @@ -190,6 +201,8 @@ production_manual: .timed_rollout_template: &timed_rollout_template <<: *rollout_template rules: + - if: '$CI_DEPLOY_FREEZE != null' + when: never - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml index a130b09c51a..1ec1aa60d88 100644 --- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml @@ -41,9 +41,9 @@ echo "Adopting Helm v2 manifests from $release" # some resource kinds must be listed explicitly https://github.com/kubernetes/kubernetes/issues/42885 for name in $(kubectl -n "$KUBE_NAMESPACE" get all,ingress,daemonset -o name -l chart="$chart"); do - kubectl annotate --overwrite "$name" meta.helm.sh/release-name="$release" - kubectl annotate --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE" - kubectl label --overwrite "$name" app.kubernetes.io/managed-by=Helm + kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-name="$release" + kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE" + kubectl label -n "$KUBE_NAMESPACE" --overwrite "$name" app.kubernetes.io/managed-by=Helm done done # migrate each release diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index c458ab6a00a..081a3a6cc78 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -16,6 +16,9 @@ stages: init: extends: .terraform:init +fmt: + extends: .terraform:fmt + validate: extends: .terraform:validate diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 39c3374e534..e696c75253e 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -20,7 +20,6 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ - - ${TF_ROOT}/.terraform.lock.hcl .init: &init stage: init diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index c30860ad174..3a70e6bc4b8 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -20,7 +20,6 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ - - ${TF_ROOT}/.terraform.lock.hcl .terraform:init: &terraform_init stage: init @@ -28,6 +27,14 @@ cache: - cd ${TF_ROOT} - gitlab-terraform init +.terraform:fmt: &terraform_fmt + stage: validate + needs: [] + script: + - cd ${TF_ROOT} + - gitlab-terraform fmt -check -recursive + allow_failure: true + .terraform:validate: &terraform_validate stage: validate script: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index dd88953b9a4..841f17767eb 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -21,7 +21,7 @@ # # The deploy stage copies the exe and msi from build stage to a network drive # You need to have the network drive mapped as Local System user for gitlab-runner service to see it -# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), +# The best way to persist the mapping is via a scheduled task # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes # place project specific paths in variables to make the rest of the script more generic diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f9798023838..72a94dcd412 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -4,6 +4,7 @@ module Gitlab module Ci class Trace include ::Gitlab::ExclusiveLeaseHelpers + include ::Gitlab::Utils::StrongMemoize include Checksummable LOCK_TTL = 10.minutes @@ -23,6 +24,8 @@ module Gitlab attr_reader :job delegate :old_trace, to: :job + delegate :can_attempt_archival_now?, :increment_archival_attempts!, + :archival_attempts_message, to: :trace_metadata def initialize(job) @job = job @@ -188,11 +191,7 @@ module Gitlab def unsafe_archive! raise ArchiveError, 'Job is not finished yet' unless job.complete? - if trace_artifact - unsafe_trace_cleanup! - - raise AlreadyArchivedError, 'Could not archive again' - end + unsafe_trace_conditionally_cleanup_before_retry! if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| @@ -212,12 +211,19 @@ module Gitlab end end - def unsafe_trace_cleanup! + def already_archived? + # TODO check checksum to ensure archive completed successfully + # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 + trace_artifact.archived_trace_exists? + end + + def unsafe_trace_conditionally_cleanup_before_retry! return unless trace_artifact - if trace_artifact.archived_trace_exists? + if already_archived? # An archive already exists, so make sure to remove the trace chunks erase_trace_chunks! + raise AlreadyArchivedError, 'Could not archive again' else # An archive already exists, but its associated file does not, so remove it trace_artifact.destroy! @@ -251,11 +257,19 @@ module Gitlab File.open(path) do |stream| # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 - job.create_job_artifacts_trace!( + trace_artifact = job.create_job_artifacts_trace!( project: job.project, file_type: :trace, file: stream, file_sha256: self.class.hexdigest(path)) + + trace_metadata.track_archival!(trace_artifact.id) + end + end + + def trace_metadata + strong_memoize(:trace_metadata) do + job.ensure_trace_metadata! end end diff --git a/lib/gitlab/ci/trace/backoff.rb b/lib/gitlab/ci/trace/backoff.rb new file mode 100644 index 00000000000..c13d88cced1 --- /dev/null +++ b/lib/gitlab/ci/trace/backoff.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Backoff class is responsible for calculating a backoff value + # for when to be able to retry archiving a build's trace + # + # Because we're updating `last_archival_attempt_at` timestamp with every + # failed archival attempt, we need to be sure that sum of the backoff values + # for 1..MAX_ATTEMPTS is under 7 days(CHUNK_REDIS_TTL). + # + class Backoff + include Gitlab::Utils::StrongMemoize + + MAX_JITTER_VALUE = 4 + + attr_reader :archival_attempts + + def initialize(archival_attempts) + @archival_attempts = archival_attempts + end + + def value + (((chunks_ttl / (3.5 * max_attempts)) * archival_attempts) / 1.hour).hours + end + + # This formula generates an increasing delay between executions + # 9.6, 19.2, 28.8, 38.4, 48.0 + a random amount of time to + # change the order of execution for the jobs. + # With maximum value for each call to rand(4), this sums up to 6.8 days + # and with minimum values is 6 days. + # + def value_with_jitter + value + jitter + end + + private + + def jitter + rand(MAX_JITTER_VALUE).hours + end + + def chunks_ttl + ::Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL + end + + def max_attempts + ::Ci::BuildTraceMetadata::MAX_ATTEMPTS + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index fdc598c025a..2d31049a0c9 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -3,7 +3,6 @@ module Gitlab module Ci class Trace - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 class Stream BUFFER_SIZE = 4096 LIMIT_SIZE = 500.kilobytes diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index ef9ba1b73c7..09c75a2b3f1 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -10,7 +10,7 @@ module Gitlab def initialize(variables = [], errors = nil) @variables = [] - @variables_by_key = {} + @variables_by_key = Hash.new { |h, k| h[k] = [] } @errors = errors variables.each { |variable| self.append(variable) } @@ -19,7 +19,7 @@ module Gitlab def append(resource) item = Collection::Item.fabricate(resource) @variables.append(item) - @variables_by_key[item[:key]] = item + @variables_by_key[item[:key]] << item self end @@ -46,7 +46,12 @@ module Gitlab end def [](key) - @variables_by_key[key] + all(key)&.last + end + + def all(key) + vars = @variables_by_key[key] + vars unless vars.empty? end def size @@ -72,7 +77,7 @@ module Gitlab match = Regexp.last_match if match[:key] # we matched variable - if variable = @variables_by_key[match[:key]] + if variable = self[match[:key]] variable.value elsif keep_undefined match[0] @@ -85,7 +90,7 @@ module Gitlab end def sort_and_expand_all(project, keep_undefined: false) - return self if Feature.disabled?(:variable_inside_variable, project) + return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb index 90a929b8a07..62637825c15 100644 --- a/lib/gitlab/ci/variables/collection/sort.rb +++ b/lib/gitlab/ci/variables/collection/sort.rb @@ -42,7 +42,7 @@ module Gitlab depends_on = var_item.depends_on return unless depends_on - depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block) + depends_on.filter_map { |var_ref_name| @collection.all(var_ref_name) }.flatten.each(&block) end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index c94fa84f608..1aa3dbc5e47 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -47,9 +47,7 @@ module Gitlab validate_job!(name, job) end - if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) - YamlProcessor::Dag.check_circular_dependencies!(@jobs) - end + YamlProcessor::Dag.check_circular_dependencies!(@jobs) end def validate_job!(name, job) @@ -103,16 +101,8 @@ module Gitlab job_stage_index = stage_index(name) dependency_stage_index = stage_index(dependency) - if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) - unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") - end - else - # A dependency might be defined later in the configuration - # with a stage that does not exist - unless dependency_stage_index.present? && dependency_stage_index < job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") - end + unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") end end |