From 143f196f8b3c40ceb7e9335a8dcc712b079519b9 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 18 Nov 2019 18:06:53 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- lib/gitlab/ci/pipeline/chain/base.rb | 2 +- lib/gitlab/ci/pipeline/chain/build.rb | 2 - lib/gitlab/ci/pipeline/chain/command.rb | 4 +- lib/gitlab/ci/pipeline/chain/config/content.rb | 59 ++++++++++++++++++++ lib/gitlab/ci/pipeline/chain/config/process.rb | 41 ++++++++++++++ .../ci/pipeline/chain/evaluate_workflow_rules.rb | 2 +- lib/gitlab/ci/pipeline/chain/populate.rb | 21 +------ .../ci/pipeline/chain/remove_unwanted_chat_jobs.rb | 6 +- lib/gitlab/ci/pipeline/chain/seed.rb | 64 ++++++++++++++++++++++ lib/gitlab/ci/pipeline/chain/validate/config.rb | 33 ----------- .../v1/rename_namespaces.rb | 10 ++-- lib/gitlab/gitaly_client/namespace_service.rb | 15 +++++ 12 files changed, 196 insertions(+), 63 deletions(-) create mode 100644 lib/gitlab/ci/pipeline/chain/config/content.rb create mode 100644 lib/gitlab/ci/pipeline/chain/config/process.rb create mode 100644 lib/gitlab/ci/pipeline/chain/seed.rb delete mode 100644 lib/gitlab/ci/pipeline/chain/validate/config.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index bab1c73e2f1..aabdf7ce47d 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Chain class Base - attr_reader :pipeline, :command + attr_reader :pipeline, :command, :config delegate :project, :current_user, to: :command diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 899df81ea5c..9662209f88e 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -22,8 +22,6 @@ module Gitlab external_pull_request: @command.external_pull_request, variables_attributes: Array(@command.variables_attributes) ) - - @pipeline.set_config_source end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 58f89a6be5e..c2df419cca0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,9 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update + :chat_data, :allow_mirror_update, + # These attributes are set by Chains during processing: + :config_content, :config_processor, :stage_seeds ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb new file mode 100644 index 00000000000..a8cd99b8e92 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content < Chain::Base + include Chain::Helpers + + def perform! + return if @command.config_content + + if content = content_from_repo + @command.config_content = content + @pipeline.config_source = :repository_source + # TODO: we should persist ci_config_path + # @pipeline.config_path = ci_config_path + elsif content = content_from_auto_devops + @command.config_content = content + @pipeline.config_source = :auto_devops_source + end + + unless @command.config_content + return error("Missing #{ci_config_path} file") + end + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + + private + + def content_from_repo + return unless project + return unless @pipeline.sha + return unless ci_config_path + + project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path) + rescue GRPC::NotFound, GRPC::Internal + nil + end + + def content_from_auto_devops + return unless project&.auto_devops_enabled? + + Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content + end + + def ci_config_path + project.ci_config_path.presence || '.gitlab-ci.yml' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb new file mode 100644 index 00000000000..731b0fdb286 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Process < Chain::Base + include Chain::Helpers + + def perform! + raise ArgumentError, 'missing config content' unless @command.config_content + + @command.config_processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + sha: @pipeline.sha, + user: current_user + } + ) + rescue Gitlab::Ci::YamlProcessor::ValidationError => ex + error(ex.message, config_error: true) + rescue => ex + Gitlab::Sentry.track_acceptable_exception(ex, extra: { + project_id: project.id, + sha: @pipeline.sha + }) + + error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})", + config_error: true) + end + + def break? + @pipeline.errors.any? || @pipeline.persisted? + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 5b46a43b725..0ee9485eebc 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -41,7 +41,7 @@ module Gitlab end def workflow_config - @pipeline.config_processor.workflow_attributes || {} + @command.config_processor.workflow_attributes || {} end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 13eca5a9d28..3a40c7b167c 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -10,29 +10,12 @@ module Gitlab PopulateError = Class.new(StandardError) def perform! - # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - - # Protect the pipeline. This is assigned in Populate instead of - # Build to prevent erroring out on ambiguous refs. - pipeline.protected = @command.protected_ref? - - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - - ## - # Gather all runtime build/stage errors - # - if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence - return error(seeds_errors.join("\n"), config_error: true) - end + raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds ## # Populate pipeline with all stages, and stages with builds. # - pipeline.stages = pipeline.stage_seeds.map(&:to_resource) + pipeline.stages = @command.stage_seeds.map(&:to_resource) if pipeline.stages.none? return error('No stages / jobs for this pipeline.') diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb index 1e09b417311..9267c72efa4 100644 --- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb @@ -6,11 +6,13 @@ module Gitlab module Chain class RemoveUnwantedChatJobs < Chain::Base def perform! - return unless pipeline.config_processor && pipeline.chat? + raise ArgumentError, 'missing config processor' unless @command.config_processor + + return unless pipeline.chat? # When scheduling a chat pipeline we only want to run the build # that matches the chat command. - pipeline.config_processor.jobs.select! do |name, _| + @command.config_processor.jobs.select! do |name, _| name.to_s == command.chat_data[:command].to_s end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb new file mode 100644 index 00000000000..2e177cfec7e --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Seed < Chain::Base + include Chain::Helpers + include Gitlab::Utils::StrongMemoize + + def perform! + raise ArgumentError, 'missing config processor' unless @command.config_processor + + # Allocate next IID. This operation must be outside of transactions of pipeline creations. + pipeline.ensure_project_iid! + + # Protect the pipeline. This is assigned in Populate instead of + # Build to prevent erroring out on ambiguous refs. + pipeline.protected = @command.protected_ref? + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + ## + # Gather all runtime build/stage errors + # + if stage_seeds_errors + return error(stage_seeds_errors.join("\n"), config_error: true) + end + + @command.stage_seeds = stage_seeds + end + + def break? + pipeline.errors.any? + end + + private + + def stage_seeds_errors + stage_seeds.flat_map(&:errors).compact.presence + end + + def stage_seeds + strong_memoize(:stage_seeds) do + seeds = stages_attributes.inject([]) do |previous_stages, attributes| + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages) + previous_stages + [seed] + end + + seeds.select(&:included?) + end + end + + def stages_attributes + @command.config_processor.stages_attributes + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb deleted file mode 100644 index 28c38cc3d18..00000000000 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Validate - class Config < Chain::Base - include Chain::Helpers - - def perform! - unless @pipeline.config_processor - unless @pipeline.ci_yaml_file - return error("Missing #{@pipeline.ci_yaml_file_path} file") - end - - if @command.save_incompleted && @pipeline.has_yaml_errors? - @pipeline.drop!(:config_error) - end - - error(@pipeline.yaml_errors) - end - end - - def break? - @pipeline.errors.any? || @pipeline.persisted? - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 3e8a9b89998..cea25967801 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -66,11 +66,13 @@ module Gitlab def move_repositories(namespace, old_full_path, new_full_path) repo_shards_for_namespace(namespace).each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage, old_full_path) + Gitlab::GitalyClient::NamespaceService.allow do + gitlab_shell.add_namespace(repository_storage, old_full_path) - unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) - message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Rails.logger.error message # rubocop:disable Gitlab/RailsLogger + unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) + message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" + Rails.logger.error message # rubocop:disable Gitlab/RailsLogger + end end end end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index f95833eed01..dbcebec3aa2 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -3,7 +3,22 @@ module Gitlab module GitalyClient class NamespaceService + extend Gitlab::TemporarilyAllow + + NamespaceServiceAccessError = Class.new(StandardError) + ALLOW_KEY = :allow_namespace + + def self.allow + temporarily_allow(ALLOW_KEY) { yield } + end + + def self.denied? + !temporarily_allowed?(ALLOW_KEY) + end + def initialize(storage) + raise NamespaceServiceAccessError if self.class.denied? + @storage = storage end -- cgit v1.2.1