Add latest changes from gitlab-org/gitlab@master

12 files changed, 180 insertions, 19 deletions

diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index 356e881a0a5..d4b3c58ec56 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -76,9 +76,17 @@ module Gitlab
         ::Feature.enabled?(:ci_job_entry_matches_all_keys)
       end
 
+      def self.lint_creates_pipeline_with_dry_run?(project)
+        ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project)
+      end
+
       def self.reset_ci_minutes_for_all_namespaces?
         ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false)
       end
+
+      def self.expand_names_for_cross_pipeline_artifacts?(project)
+        ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project)
+      end
     end
   end
 end
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
new file mode 100644
index 00000000000..468f3bc4689
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class CancelPendingPipelines < Chain::Base
+          include Chain::Helpers
+
+          def perform!
+            return unless project.auto_cancel_pending_pipelines?
+
+            Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables|
+              cancelables.find_each do |cancelable|
+                cancelable.auto_cancel_running(pipeline)
+              end
+            end
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def auto_cancelable_pipelines
+            project.ci_pipelines
+              .where(ref: pipeline.ref)
+              .where.not(id: pipeline.same_family_pipeline_ids)
+              .where.not(sha: project.commit(pipeline.ref).try(:id))
+              .alive_or_scheduled
+              .with_only_interruptible_builds
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 74b28b181bc..dbaa6951e64 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -10,7 +10,7 @@ module Gitlab
           :trigger_request, :schedule, :merge_request, :external_pull_request,
           :ignore_skip_ci, :save_incompleted,
           :seeds_block, :variables_attributes, :push_options,
-          :chat_data, :allow_mirror_update, :bridge, :content,
+          :chat_data, :allow_mirror_update, :bridge, :content, :dry_run,
           # These attributes are set by Chains during processing:
           :config_content, :config_processor, :stage_seeds
         ) do
@@ -22,6 +22,8 @@ module Gitlab
             end
           end
 
+          alias_method :dry_run?, :dry_run
+
           def branch_exists?
             strong_memoize(:is_branch) do
               project.repository.branch_exists?(ref)
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
index 3dd216b33d1..9954aedc4b7 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
@@ -12,7 +12,6 @@ module Gitlab
               def content
                 strong_memoize(:content) do
                   next unless command.content.present?
-                  raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build?
 
                   command.content
                 end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index aba7dab508d..d7271df1694 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -6,13 +6,13 @@ module Gitlab
       module Chain
         module Helpers
           def error(message, config_error: false, drop_reason: nil)
-            if config_error && command.save_incompleted
+            if config_error
               drop_reason = :config_error
               pipeline.yaml_errors = message
             end
 
             pipeline.add_error_message(message)
-            pipeline.drop!(drop_reason) if drop_reason
+            pipeline.drop!(drop_reason) if drop_reason && persist_pipeline?
 
             # TODO: consider not to rely on AR errors directly as they can be
             # polluted with other unrelated errors (e.g. state machine)
@@ -23,6 +23,10 @@ module Gitlab
           def warning(message)
             pipeline.add_warning_message(message)
           end
+
+          def persist_pipeline?
+            command.save_incompleted && !pipeline.readonly?
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
new file mode 100644
index 00000000000..0d7449813b4
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Metrics < Chain::Base
+          def perform!
+            counter.increment(source: @pipeline.source)
+          end
+
+          def break?
+            false
+          end
+
+          def counter
+            ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
new file mode 100644
index 00000000000..98e9e1479bc
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Pipeline
+          # After pipeline has been successfully created we can start processing it.
+          class Process < Chain::Base
+            def perform!
+              ::Ci::ProcessPipelineService
+                .new(@pipeline)
+                .execute(nil, initial_process: true)
+            end
+
+            def break?
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 204c7725214..dc648568129 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -9,30 +9,21 @@ module Gitlab
             @pipeline = pipeline
             @command = command
             @sequence = sequence
-            @completed = []
             @start = Time.now
           end
 
           def build!
-            @sequence.each do |chain|
-              step = chain.new(@pipeline, @command)
+            @sequence.each do |step_class|
+              step = step_class.new(@pipeline, @command)
 
               step.perform!
               break if step.break?
-
-              @completed.push(step)
             end
 
-            @pipeline.tap do
-              yield @pipeline, self if block_given?
-
-              @command.observe_creation_duration(Time.now - @start)
-              @command.observe_pipeline_size(@pipeline)
-            end
-          end
+            @command.observe_creation_duration(Time.now - @start)
+            @command.observe_pipeline_size(@pipeline)
 
-          def complete?
-            @completed.size == @sequence.size
+            @pipeline
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb
new file mode 100644
index 00000000000..0e9add4ee74
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        # During the dry run we don't want to persist the pipeline and skip
+        # all the other steps that operate on a persisted context.
+        # This causes the chain to break at this point.
+        class StopDryRun < Chain::Base
+          def perform!
+            # no-op
+          end
+
+          def break?
+            @command.dry_run?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 649da745eea..ef6981eeb4d 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -36,6 +36,24 @@ module Gitlab
             Gitlab::Metrics.counter(name, comment)
           end
         end
+
+        def pipelines_created_counter
+          strong_memoize(:pipelines_created_count) do
+            name = :pipelines_created_total
+            comment = 'Counter of pipelines created'
+
+            Gitlab::Metrics.counter(name, comment)
+          end
+        end
+
+        def pipelines_created_counter
+          strong_memoize(:pipelines_created_count) do
+            name = :pipelines_created_total
+            comment = 'Counter of pipelines created'
+
+            Gitlab::Metrics.counter(name, comment)
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 21f837c58bb..d6681347f42 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -220,5 +220,33 @@ module Gitlab
         end
       end
     end
+
+    class LimitedEncoder
+      LimitExceeded = Class.new(StandardError)
+
+      # Generates JSON for an object or raise an error if the resulting json string is too big
+      #
+      # @param object [Hash, Array, Object] must be hash, array, or an object that responds to .to_h or .to_json
+      # @param limit [Integer] max size of the resulting json string
+      # @return [String]
+      # @raise [LimitExceeded] if the resulting json string is bigger than the specified limit
+      def self.encode(object, limit: 25.megabytes)
+        return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder)
+
+        buffer = []
+        buffer_size = 0
+
+        ::Yajl::Encoder.encode(object) do |data_chunk|
+          chunk_size = data_chunk.bytesize
+
+          raise LimitExceeded if buffer_size + chunk_size > limit
+
+          buffer << data_chunk
+          buffer_size += chunk_size
+        end
+
+        buffer.join('')
+      end
+    end
   end
 end
diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb
index 08ed6599a6e..d335a434335 100644
--- a/lib/gitlab/static_site_editor/config.rb
+++ b/lib/gitlab/static_site_editor/config.rb
@@ -3,7 +3,7 @@
 module Gitlab
   module StaticSiteEditor
     class Config
-      SUPPORTED_EXTENSIONS = %w[.md .md.erb].freeze
+      SUPPORTED_EXTENSIONS = %w[.md].freeze
 
       def initialize(repository, ref, file_path, return_url)
         @repository = repository
@@ -42,6 +42,8 @@ module Gitlab
       end
 
       def extension_supported?
+        return true if file_path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project)
+
         SUPPORTED_EXTENSIONS.any? { |ext| file_path.end_with?(ext) }
       end