diff options
Diffstat (limited to 'lib/gitlab/ci')
40 files changed, 726 insertions, 130 deletions
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index cbda3808b86..0373a12ab69 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -37,16 +37,13 @@ module Gitlab flush_current_line - # TODO: replace OpenStruct with a better type - # https://gitlab.com/gitlab-org/gitlab/issues/34305 - OpenStruct.new( + Gitlab::Ci::Ansi2json::Result.new( lines: @lines, state: @state.encode, append: append, truncated: truncated, offset: start_offset, - size: stream.tell - start_offset, - total: stream.size + stream: stream ) end diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index d428680fb2a..79b42a5f5bf 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -94,7 +94,7 @@ module Gitlab def on_38(stack) { fg: fg_color_256(stack) } end - def on_39(_) { fg: fg_color(9) } end + def on_39(_) { fg: nil } end def on_40(_) { bg: bg_color(0) } end @@ -114,8 +114,7 @@ module Gitlab def on_48(stack) { bg: bg_color_256(stack) } end - # TODO: all the x9 never get called? - def on_49(_) { fg: fg_color(9) } end + def on_49(_) { bg: nil } end def on_90(_) { fg: fg_color(0, 'l') } end diff --git a/lib/gitlab/ci/ansi2json/result.rb b/lib/gitlab/ci/ansi2json/result.rb new file mode 100644 index 00000000000..9b573882a52 --- /dev/null +++ b/lib/gitlab/ci/ansi2json/result.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Convertion result object class +module Gitlab + module Ci + module Ansi2json + class Result + attr_reader :lines, :state, :append, :truncated, :offset, :size, :total + + def initialize(lines:, state:, append:, truncated:, offset:, stream:) + @lines = lines + @state = state + @append = append + @truncated = truncated + @offset = offset + @size = stream.tell - offset + @total = stream.size + end + end + end + end +end diff --git a/lib/gitlab/ci/ansi2json/style.rb b/lib/gitlab/ci/ansi2json/style.rb index 77f61178b37..4d38ea55866 100644 --- a/lib/gitlab/ci/ansi2json/style.rb +++ b/lib/gitlab/ci/ansi2json/style.rb @@ -61,9 +61,9 @@ module Gitlab case when changes[:reset] reset! - when changes[:fg] + when changes.key?(:fg) @fg = changes[:fg] - when changes[:bg] + when changes.key?(:bg) @bg = changes[:bg] when changes[:enable] @mask |= changes[:enable] diff --git a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb index 9950e1dec55..465877871ea 100644 --- a/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb +++ b/lib/gitlab/ci/build/prerequisite/kubernetes_namespace.rb @@ -8,7 +8,7 @@ module Gitlab def unmet? deployment_cluster.present? && deployment_cluster.managed? && - missing_namespace? + (missing_namespace? || need_knative_version_role_binding?) end def complete! @@ -23,6 +23,10 @@ module Gitlab kubernetes_namespace.nil? || kubernetes_namespace.service_account_token.blank? end + def need_knative_version_role_binding? + !knative_serving_namespace.nil? && knative_version_role_binding.nil? + end + def deployment_cluster build.deployment&.cluster end @@ -31,6 +35,22 @@ module Gitlab build.deployment.environment end + def knative_serving_namespace + strong_memoize(:knative_serving_namespace) do + Clusters::KnativeServingNamespaceFinder.new( + deployment_cluster + ).execute + end + end + + def knative_version_role_binding + strong_memoize(:knative_version_role_binding) do + Clusters::KnativeVersionRoleBindingFinder.new( + deployment_cluster + ).execute + end + end + def kubernetes_namespace strong_memoize(:kubernetes_namespace) do Clusters::KubernetesNamespaceFinder.new( @@ -43,12 +63,33 @@ module Gitlab end def create_namespace + namespace = kubernetes_namespace || build_namespace_record + + return if conflicting_ci_namespace_requested?(namespace) + Clusters::Kubernetes::CreateOrUpdateNamespaceService.new( cluster: deployment_cluster, - kubernetes_namespace: kubernetes_namespace || build_namespace_record + kubernetes_namespace: namespace ).execute end + ## + # A namespace can only be specified via gitlab-ci.yml + # for unmanaged clusters, as we currently have no way + # of preventing a job requesting a namespace it + # shouldn't have access to. + # + # To make this clear, we fail the build instead of + # silently using a namespace other than the one + # explicitly specified. + # + # Support for managed clusters will be added in + # https://gitlab.com/gitlab-org/gitlab/issues/38054 + def conflicting_ci_namespace_requested?(namespace_record) + build.expanded_kubernetes_namespace.present? && + namespace_record.namespace != build.expanded_kubernetes_namespace + end + def build_namespace_record Clusters::BuildKubernetesNamespaceService.new( deployment_cluster, diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9c1e6277e95..38ab3475d01 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -67,11 +67,11 @@ module Gitlab build_config(config) rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e - track_exception(e) + track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message rescue Gitlab::Ci::Config::External::Context::TimeoutError => e - track_exception(e) + track_and_raise_for_dev_exception(e) raise Config::ConfigError, TIMEOUT_MESSAGE end @@ -94,8 +94,8 @@ module Gitlab user: user) end - def track_exception(error) - Gitlab::Sentry.track_exception(error, extra: @context.sentry_payload) + def track_and_raise_for_dev_exception(error) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end # Overriden in EE diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index 83127bde6e4..88db17a75da 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -14,7 +14,8 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable ALLOWED_KEYS = %i[before_script image services - after_script cache interruptible].freeze + after_script cache interruptible + timeout retry tags artifacts].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -40,11 +41,27 @@ module Gitlab description: 'Configure caching between build jobs.', inherit: true - entry :interruptible, Entry::Boolean, + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible default value.', inherit: false - helpers :before_script, :image, :services, :after_script, :cache, :interruptible + entry :timeout, Entry::Timeout, + description: 'Set jobs default timeout.', + inherit: false + + entry :retry, Entry::Retry, + description: 'Set retry default value.', + inherit: false + + entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + description: 'Set the default tags.', + inherit: false + + entry :artifacts, Entry::Artifacts, + description: 'Default artifacts.', + inherit: false + + helpers :before_script, :image, :services, :after_script, :cache private diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 5a13fd18504..fc62cca58ff 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -8,9 +8,11 @@ module Gitlab # Entry that represents an environment. # class Environment < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze + + entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' validations do validate do @@ -46,6 +48,8 @@ module Gitlab allow_nil: true validates :on_stop, type: String, allow_nil: true + validates :kubernetes, type: Hash, allow_nil: true + validates :auto_stop_in, duration: true, allow_nil: true end end @@ -73,6 +77,14 @@ module Gitlab value[:on_stop] end + def kubernetes + value[:kubernetes] + end + + def auto_stop_in + value[:auto_stop_in] + end + def value case @config when String then { name: @config, action: 'start' } @@ -80,6 +92,10 @@ module Gitlab else {} end end + + def skip_config_hash_validation? + true + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index c75ae87a985..6a55b8cda57 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -36,7 +36,6 @@ module Gitlab if: :has_rules? with_options allow_nil: true do - validates :tags, array_of_strings: true validates :allow_failure, boolean: true validates :parallel, numericality: { only_integer: true, greater_than_or_equal_to: 2, @@ -46,14 +45,12 @@ module Gitlab message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } - validates :timeout, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } - validates :dependencies, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true end - validates :start_in, duration: { limit: '1 day' }, if: :delayed? + validates :start_in, duration: { limit: '1 week' }, if: :delayed? validates :start_in, absence: true, if: -> { has_rules? || !delayed? } validate do @@ -99,13 +96,29 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true - entry :interruptible, Entry::Boolean, + entry :interruptible, ::Gitlab::Config::Entry::Boolean, description: 'Set jobs interruptible value.', inherit: true + entry :timeout, Entry::Timeout, + description: 'Timeout duration of this job.', + inherit: true + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.', + inherit: true + + entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings, + description: 'Set the tags.', + inherit: true + + entry :artifacts, Entry::Artifacts, + description: 'Artifacts configuration for this job.', + inherit: true + entry :only, Entry::Policy, description: 'Refs policy this job will be executed for.', - default: Entry::Policy::DEFAULT_ONLY, + default: ::Gitlab::Ci::Config::Entry::Policy::DEFAULT_ONLY, inherit: false entry :except, Entry::Policy, @@ -121,17 +134,13 @@ module Gitlab entry :needs, Entry::Needs, description: 'Needs configuration for this job.', - metadata: { allowed_needs: %i[job] }, + metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false entry :variables, Entry::Variables, description: 'Environment variables available for this job.', inherit: false - entry :artifacts, Entry::Artifacts, - description: 'Artifacts configuration for this job.', - inherit: false - entry :environment, Entry::Environment, description: 'Environment configuration for this job.', inherit: false @@ -140,10 +149,6 @@ module Gitlab description: 'Coverage configuration for this job.', inherit: false - entry :retry, Entry::Retry, - description: 'Retry configuration for this job.', - inherit: false - helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :environment, :coverage, :retry, :rules, @@ -170,11 +175,18 @@ module Gitlab @entries.delete(:type) - # This is something of a hack, see issue for details: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/67150 - if !only_defined? && has_rules? - @entries.delete(:only) - @entries.delete(:except) + has_workflow_rules = deps&.workflow&.has_rules? + + # If workflow:rules: or rules: are used + # they are considered not compatible + # with `only/except` defaults + # + # Context: https://gitlab.com/gitlab-org/gitlab/merge_requests/21742 + if has_rules? || has_workflow_rules + # Remove only/except defaults + # defaults are not considered as defined + @entries.delete(:only) unless only_defined? + @entries.delete(:except) unless except_defined? end end end diff --git a/lib/gitlab/ci/config/entry/kubernetes.rb b/lib/gitlab/ci/config/entry/kubernetes.rb new file mode 100644 index 00000000000..2f1595d4437 --- /dev/null +++ b/lib/gitlab/ci/config/entry/kubernetes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Kubernetes < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[namespace].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + validates :namespace, type: String, presence: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index b6db546d8ff..abfffb7a5ed 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -5,9 +5,12 @@ module Gitlab class Config module Entry class Need < ::Gitlab::Config::Entry::Simplifiable - strategy :Job, if: -> (config) { config.is_a?(String) } + strategy :JobString, if: -> (config) { config.is_a?(String) } - class Job < ::Gitlab::Config::Entry::Node + strategy :JobHash, + if: -> (config) { config.is_a?(Hash) && config.key?(:job) && !(config.key?(:project) || config.key?(:ref)) } + + class JobString < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable validations do @@ -20,7 +23,30 @@ module Gitlab end def value - { name: @config } + { name: @config, artifacts: true } + end + end + + class JobHash < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[job artifacts].freeze + attributes :job, :artifacts + + validations do + validates :config, presence: true + validates :config, allowed_keys: ALLOWED_KEYS + validates :job, type: String, presence: true + validates :artifacts, boolean: true, allow_nil: true + end + + def type + :job + end + + def value + { name: job, artifacts: artifacts || artifacts.nil? } end end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index 28452aaaa16..5301c453ed4 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -53,3 +53,5 @@ module Gitlab end end end + +::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs') diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 25fb278d9b8..12dd942fc1c 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -67,7 +67,7 @@ module Gitlab entry :workflow, Entry::Workflow, description: 'List of evaluable rules to determine Pipeline status' - helpers :default, :jobs, :stages, :types, :variables + helpers :default, :jobs, :stages, :types, :variables, :workflow delegate :before_script_value, :image_value, @@ -106,6 +106,10 @@ module Gitlab self[:default] end + def workflow + self[:workflow] if workflow_defined? + end + private # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/timeout.rb index 10619ef9f8d..0bffa9340de 100644 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ b/lib/gitlab/ci/config/entry/timeout.rb @@ -7,11 +7,11 @@ module Gitlab ## # Entry that represents the interrutible value. # - class Boolean < ::Gitlab::Config::Entry::Node + class Timeout < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable validations do - validates :config, boolean: true + validates :config, duration: { limit: ChronicDuration.output(Project::MAX_BUILD_TIMEOUT) } end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index a51a3fbdcd2..1d9007c9b9b 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -18,6 +18,10 @@ module Gitlab entry :rules, Entry::Rules, description: 'List of evaluable Rules to determine Pipeline status.', metadata: { allowed_when: %w[always never] } + + def has_rules? + @config.try(:key?, :rules) + end end end end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index e714ef225f5..1139efee9e8 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -44,7 +44,7 @@ module Gitlab if all_job_names = parallelized_jobs[job_need_name] all_job_names.map do |job_name| - { name: job_name } + job_need.merge(name: job_name) end else job_need diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a8cd99b8e92..d4b7444005e 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -8,21 +8,28 @@ module Gitlab class Content < Chain::Base include Chain::Helpers - def perform! - return if @command.config_content - - if content = content_from_repo - @command.config_content = content - @pipeline.config_source = :repository_source - # TODO: we should persist ci_config_path - # @pipeline.config_path = ci_config_path - elsif content = content_from_auto_devops - @command.config_content = content - @pipeline.config_source = :auto_devops_source - end + SOURCES = [ + Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, + Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, + Gitlab::Ci::Pipeline::Chain::Config::Content::Remote, + Gitlab::Ci::Pipeline::Chain::Config::Content::AutoDevops + ].freeze + + LEGACY_SOURCES = [ + Gitlab::Ci::Pipeline::Chain::Config::Content::Runtime, + Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyRepository, + Gitlab::Ci::Pipeline::Chain::Config::Content::LegacyAutoDevops + ].freeze - unless @command.config_content - return error("Missing #{ci_config_path} file") + def perform! + if config = find_config + # TODO: we should persist config_content + # @pipeline.config_content = config.content + @command.config_content = config.content + @pipeline.config_source = config.source + else + error('Missing CI config file') end end @@ -32,24 +39,21 @@ module Gitlab private - def content_from_repo - return unless project - return unless @pipeline.sha - return unless ci_config_path + def find_config + sources.each do |source| + config = source.new(@pipeline, @command) + return config if config.exists? + end - project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) - rescue GRPC::NotFound, GRPC::Internal nil end - def content_from_auto_devops - return unless project&.auto_devops_enabled? - - Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content - end - - def ci_config_path - project.ci_config_path.presence || '.gitlab-ci.yml' + def sources + if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true) + SOURCES + else + LEGACY_SOURCES + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb new file mode 100644 index 00000000000..e9bcc67de9c --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class AutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + YAML.dump('include' => [{ 'template' => template.full_name }]) + end + end + + def source + :auto_devops_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb new file mode 100644 index 00000000000..8a19e433483 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class ExternalProject < Source + def content + strong_memoize(:content) do + next unless external_project_path? + + path_file, path_project = ci_config_path.split('@', 2) + YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }]) + end + end + + def source + :external_project_source + end + + private + + # Example: path/to/.gitlab-ci.yml@another-group/another-project + def external_project_path? + ci_config_path =~ /\A.+(yml|yaml)@.+\z/ + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb new file mode 100644 index 00000000000..c4cef356628 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class LegacyAutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps') + template.content + end + end + + def source + :auto_devops_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb new file mode 100644 index 00000000000..fa4a97c6880 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/legacy_repository.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class LegacyRepository < Source + def content + strong_memoize(:content) do + next unless project + next unless @pipeline.sha + next unless ci_config_path + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + + def source + :repository_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb new file mode 100644 index 00000000000..dcc336b8929 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Remote < Source + def content + strong_memoize(:content) do + next unless ci_config_path =~ URI.regexp(%w[http https]) + + YAML.dump('include' => [{ 'remote' => ci_config_path }]) + end + end + + def source + :remote_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/repository.rb b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb new file mode 100644 index 00000000000..0752b099d3d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/repository.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Repository < Source + def content + strong_memoize(:content) do + next unless file_in_repository? + + YAML.dump('include' => [{ 'local' => ci_config_path }]) + end + end + + def source + :repository_source + end + + private + + def file_in_repository? + return unless project + return unless @pipeline.sha + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path).present? + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb b/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb new file mode 100644 index 00000000000..4811d3d913d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/runtime.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Runtime < Source + def content + @command.config_content + end + + def source + # The only case when this source is used is when the config content + # is passed in as parameter to Ci::CreatePipelineService. + # This would only occur with parent/child pipelines which is being + # implemented. + # TODO: change source to return :runtime_source + # https://gitlab.com/gitlab-org/gitlab/merge_requests/21041 + + nil + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb new file mode 100644 index 00000000000..3389187473b --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/source.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Source + include Gitlab::Utils::StrongMemoize + + DEFAULT_YAML_FILE = '.gitlab-ci.yml' + + def initialize(pipeline, command) + @pipeline = pipeline + @command = command + end + + def exists? + strong_memoize(:exists) do + content.present? + end + end + + def content + raise NotImplementedError + end + + def source + raise NotImplementedError + end + + def project + @project ||= @pipeline.project + end + + def ci_config_path + @ci_config_path ||= project.ci_config_path.presence || DEFAULT_YAML_FILE + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 731b0fdb286..09d1b0edc93 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -21,10 +21,10 @@ module Gitlab rescue Gitlab::Ci::YamlProcessor::ValidationError => ex error(ex.message, config_error: true) rescue => ex - Gitlab::Sentry.track_acceptable_exception(ex, extra: { + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, sha: @pipeline.sha - }) + ) error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})", config_error: true) diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 0ee9485eebc..81f5733b279 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -9,7 +9,13 @@ module Gitlab include Chain::Helpers def perform! - return unless Feature.enabled?(:workflow_rules, @pipeline.project) + unless feature_enabled? + if has_workflow_rules? + error("Workflow rules are disabled", config_error: true) + end + + return + end unless workflow_passed? error('Pipeline filtered out by workflow rules.') @@ -17,13 +23,15 @@ module Gitlab end def break? - return false unless Feature.enabled?(:workflow_rules, @pipeline.project) - - !workflow_passed? + @pipeline.errors.any? || @pipeline.persisted? end private + def feature_enabled? + Feature.enabled?(:workflow_rules, @pipeline.project, default_enabled: true) + end + def workflow_passed? strong_memoize(:workflow_passed) do workflow_rules.evaluate(@pipeline, global_context).pass? @@ -40,6 +48,10 @@ module Gitlab @pipeline, yaml_variables: workflow_config[:yaml_variables]) end + def has_workflow_rules? + workflow_config[:rules].present? + end + def workflow_config @command.config_processor.workflow_attributes || {} end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 8ccb1066575..982ecc0ff51 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -5,12 +5,13 @@ module Gitlab module Pipeline module Chain module Helpers - def error(message, config_error: false) + def error(message, config_error: false, drop_reason: nil) if config_error && command.save_incompleted + drop_reason = :config_error pipeline.yaml_errors = message - pipeline.drop!(:config_error) end + pipeline.drop!(drop_reason) if drop_reason pipeline.errors.add(:base, message) end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb new file mode 100644 index 00000000000..44dc333a6a1 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class External < Chain::Base + include Chain::Helpers + + InvalidResponseCode = Class.new(StandardError) + + VALIDATION_REQUEST_TIMEOUT = 5 + + def perform! + error('External validation failed', drop_reason: :external_validation_failure) unless validate_external + end + + def break? + @pipeline.errors.any? + end + + private + + def validate_external + return true unless validation_service_url + + # 200 - accepted + # 4xx - not accepted + # everything else - accepted and logged + response_code = validate_service_request.code + case response_code + when 200 + true + when 400..499 + false + else + raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" + end + rescue => ex + Gitlab::ErrorTracking.track_exception(ex) + + true + end + + def validate_service_request + Gitlab::HTTP.post( + validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, + body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes) + ) + end + + def validation_service_url + ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_payload(pipeline, stages_attributes) + { + project: { + id: pipeline.project.id, + path: pipeline.project.full_path + }, + user: { + id: pipeline.user.id, + username: pipeline.user.username, + email: pipeline.user.email + }, + pipeline: { + sha: pipeline.sha, + ref: pipeline.ref, + type: pipeline.source + }, + builds: builds_validation_payload(stages_attributes) + }.to_json + end + + def builds_validation_payload(stages_attributes) + stages_attributes.map { |stage| stage[:builds] }.flatten + .map(&method(:build_validation_payload)) + end + + def build_validation_payload(build) + { + name: build[:name], + stage: build[:stage], + image: build.dig(:options, :image, :name), + services: build.dig(:options, :services)&.map { |service| service[:name] }, + script: [ + build.dig(:options, :before_script), + build.dig(:options, :script), + build.dig(:options, :after_script) + ].flatten.compact + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index dce56b22666..590c7f4d1dd 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -10,7 +10,7 @@ module Gitlab delegate :dig, to: :@seed_attributes # When the `ci_dag_limit_needs` is enabled it uses the lower limit - LOW_NEEDS_LIMIT = 5 + LOW_NEEDS_LIMIT = 10 HARD_NEEDS_LIMIT = 50 def initialize(pipeline, attributes, previous_stages) diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index b0a79950667..426f0238f9d 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -15,15 +15,15 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter - - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.0.0/index.js - mkdir sitespeed-results - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:11.2.0 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:11.2.0 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" fi - mv sitespeed-results/data/performance.json performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index a60b00b2ee8..1708984c1cb 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,6 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/security-products/codequality:0.85.5" script: - | if ! docker info &>/dev/null; then @@ -14,11 +15,12 @@ code_quality: export DOCKER_HOST='tcp://localhost:2375' fi fi + - docker pull --quiet "$CODE_QUALITY_IMAGE" - docker run --env SOURCE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:12-5-stable" /code + "$CODE_QUALITY_IMAGE" /code artifacts: reports: codequality: gl-code-quality-report.json diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 738be44d5f4..d20d04425f6 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.7.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.8.3" review: extends: .auto-deploy diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml new file mode 100644 index 00000000000..9a5b0f79ecf --- /dev/null +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -0,0 +1,16 @@ +apply: + stage: deploy + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.3.0" + environment: + name: production + variables: + TILLER_NAMESPACE: gitlab-managed-apps + GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml + INGRESS_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/ingress/values.yaml + SENTRY_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/sentry/values.yaml + script: + - kubectl get namespace "$TILLER_NAMESPACE" || kubectl create namespace "$TILLER_NAMESPACE" + - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml + only: + refs: + - master diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 9a3ecd1c34f..975cb3b7698 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -1,5 +1,16 @@ -# Full project: https://gitlab.com/pages/hugo -image: dettmering/hugo-build +--- +# All available Hugo versions are listed here: +# https://gitlab.com/pages/hugo/container_registry +image: registry.gitlab.com/pages/hugo:latest + +variables: + GIT_SUBMODULE_STRATEGY: recursive + +test: + script: + - hugo + except: + - master pages: script: @@ -9,9 +20,3 @@ pages: - public only: - master - -test: - script: - - hugo - except: - - master diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index ef2fc561201..f708e95c2cf 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ variables: - CS_MAJOR_VERSION: 1 + CS_MAJOR_VERSION: 2 container_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 4993d22d400..d73f6ccdb3f 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: DS_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - DS_DEFAULT_ANALYZERS: "gemnasium, retire.js, gemnasium-python, gemnasium-maven, bundler-audit" + DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_MAJOR_VERSION: 2 DS_DISABLE_DIND: "false" @@ -43,15 +43,17 @@ dependency_scanning: DS_ANALYZER_IMAGE_TAG \ DS_DEFAULT_ANALYZERS \ DS_EXCLUDED_PATHS \ - DEP_SCAN_DISABLE_REMOTE_CHECKS \ DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_VERSION \ DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + PIP_REQUIREMENTS_FILE \ MAVEN_CLI_OPTS \ + BUNDLER_AUDIT_UPDATE_DISABLED \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ @@ -70,7 +72,7 @@ dependency_scanning: - $DEPENDENCY_SCANNING_DISABLED - $DS_DISABLE_DIND == 'true' -.analyzer: +.ds-analyzer: extends: dependency_scanning services: [] except: @@ -80,7 +82,7 @@ dependency_scanning: - /analyzer run gemnasium-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION" only: @@ -90,7 +92,7 @@ gemnasium-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php/ gemnasium-maven-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" only: @@ -100,7 +102,7 @@ gemnasium-maven-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjava\b/ gemnasium-python-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" only: @@ -110,7 +112,7 @@ gemnasium-python-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ bundler-audit-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION" only: @@ -120,7 +122,7 @@ bundler-audit-dependency_scanning: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ retire-js-dependency_scanning: - extends: .analyzer + extends: .ds-analyzer image: name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION" only: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index c81b4efddbc..34d84138a8b 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -6,9 +6,10 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "false" + SCAN_KUBERNETES_MANIFESTS: "false" sast: stage: test @@ -49,7 +50,7 @@ sast: - $SAST_DISABLED - $SAST_DISABLE_DIND == 'true' -.analyzer: +.sast-analyzer: extends: sast services: [] except: @@ -59,7 +60,7 @@ sast: - /analyzer run bandit-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" only: @@ -69,7 +70,7 @@ bandit-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/ brakeman-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" only: @@ -79,7 +80,7 @@ brakeman-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/ eslint-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" only: @@ -89,7 +90,7 @@ eslint-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ flawfinder-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" only: @@ -98,8 +99,18 @@ flawfinder-sast: $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ && $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\+\+|c)\b/ +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + only: + variables: + - $GITLAB_FEATURES =~ /\bsast\b/ && + $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && + $SCAN_KUBERNETES_MANIFESTS == 'true' + gosec-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" only: @@ -109,7 +120,7 @@ gosec-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/ nodejs-scan-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" only: @@ -119,7 +130,7 @@ nodejs-scan-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/ phpcs-security-audit-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" only: @@ -129,7 +140,7 @@ phpcs-security-audit-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /php/ pmd-apex-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" only: @@ -139,7 +150,7 @@ pmd-apex-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /apex/ secrets-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" only: @@ -148,7 +159,7 @@ secrets-sast: $SAST_DEFAULT_ANALYZERS =~ /secrets/ security-code-scan-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" only: @@ -158,7 +169,7 @@ security-code-scan-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/ sobelow-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" only: @@ -168,7 +179,7 @@ sobelow-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /elixir/ spotbugs-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" only: @@ -178,7 +189,7 @@ spotbugs-sast: $CI_PROJECT_REPOSITORY_LANGUAGES =~ /java\b/ tslint-sast: - extends: .analyzer + extends: .sast-analyzer image: name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG" only: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index eced181e966..e6097ae322e 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -11,7 +11,7 @@ performance: image: docker:git variables: URL: https://example.com - SITESPEED_VERSION: 6.3.1 + SITESPEED_VERSION: 11.2.0 SITESPEED_OPTIONS: '' services: - docker:stable-dind diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 833c545fc5b..27cd4f5fd6b 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -9,6 +9,12 @@ module Gitlab attr_reader :stages, :jobs + ResultWithErrors = Struct.new(:content, :errors) do + def valid? + errors.empty? + end + end + def initialize(config, opts = {}) @ci_config = Gitlab::Ci::Config.new(config, **opts) @config = @ci_config.to_hash @@ -22,6 +28,18 @@ module Gitlab raise ValidationError, e.message end + def self.new_with_validation_errors(content, opts = {}) + return ResultWithErrors.new('', ['Please provide content of .gitlab-ci.yml']) if content.blank? + + config = Gitlab::Ci::Config.new(content, **opts) + return ResultWithErrors.new("", config.errors) unless config.valid? + + config = Gitlab::Ci::YamlProcessor.new(content, opts) + ResultWithErrors.new(config, []) + rescue ValidationError, Gitlab::Ci::Config::ConfigError => e + ResultWithErrors.new('', [e.message]) + end + def builds @jobs.map do |name, _| build_attributes(name) @@ -42,6 +60,8 @@ module Gitlab yaml_variables: transform_to_yaml_variables(job_variables(name)), needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], + only: job[:only], + except: job[:except], rules: job[:rules], cache: job[:cache], options: { @@ -49,6 +69,7 @@ module Gitlab services: job[:services], artifacts: job[:artifacts], dependencies: job[:dependencies], + cross_dependencies: job.dig(:needs, :cross_dependency), job_timeout: job[:timeout], before_script: job[:before_script], script: job[:script], @@ -71,13 +92,7 @@ module Gitlab def stages_attributes @stages.uniq.map do |stage| - seeds = stage_builds_attributes(stage).map do |attributes| - job = @jobs.fetch(attributes[:name].to_sym) - - attributes - .merge(only: job.fetch(:only, {})) - .merge(except: job.fetch(:except, {})) - end + seeds = stage_builds_attributes(stage) { name: stage, index: @stages.index(stage), builds: seeds } end |