Introduce default: for gitlab-ci.yml

This moves all existing `image/services/before_script/variables`
into `default:`. This allows us to easily add a default and
top-level entries. `default`: is keep backward compatible: to
be considered to be job if `default:script:` is specified. This
behavior should be removed.

All existing `image/services/before_script/variables` are properly
handled in root context.

10 files changed, 314 insertions, 131 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index a0275515906..7aeac11df55 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -14,23 +14,25 @@ module Gitlab
         External::Processor::IncludeError
       ].freeze
 
+      attr_reader :root
+
       def initialize(config, project: nil, sha: nil, user: nil)
         @config = Config::Extendable
           .new(build_config(config, project: project, sha: sha, user: user))
           .to_hash
 
-        @global = Entry::Global.new(@config)
-        @global.compose!
+        @root = Entry::Root.new(@config)
+        @root.compose!
       rescue *rescue_errors => e
         raise Config::ConfigError, e.message
       end
 
       def valid?
-        @global.valid?
+        @root.valid?
       end
 
       def errors
-        @global.errors
+        @root.errors
       end
 
       def to_hash
@@ -40,36 +42,16 @@ module Gitlab
       ##
       # Temporary method that should be removed after refactoring
       #
-      def before_script
-        @global.before_script_value
-      end
-
-      def image
-        @global.image_value
-      end
-
-      def services
-        @global.services_value
-      end
-
-      def after_script
-        @global.after_script_value
-      end
-
       def variables
-        @global.variables_value
+        root.variables_value
       end
 
       def stages
-        @global.stages_value
-      end
-
-      def cache
-        @global.cache_value
+        root.stages_value
       end
 
       def jobs
-        @global.jobs_value
+        root.jobs_value
       end
 
       private
diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb
new file mode 100644
index 00000000000..6200d7c7f87
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/default.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # This class represents a default entry
+        # Entry containing default values for all jobs
+        # defined in configuration file.
+        #
+        class Default < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+
+          DuplicateError = Class.new(Gitlab::Config::Loader::FormatError)
+
+          ALLOWED_KEYS = %i[before_script image services
+                            after_script cache].freeze
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+          end
+
+          entry :before_script, Entry::Script,
+            description: 'Script that will be executed before each job.',
+            inherit: true
+
+          entry :image, Entry::Image,
+            description: 'Docker image that will be used to execute jobs.',
+            inherit: true
+
+          entry :services, Entry::Services,
+            description: 'Docker images that will be linked to the container.',
+            inherit: true
+
+          entry :after_script, Entry::Script,
+            description: 'Script that will be executed after each job.',
+            inherit: true
+
+          entry :cache, Entry::Cache,
+            description: 'Configure caching between build jobs.',
+            inherit: true
+
+          helpers :before_script, :image, :services, :after_script, :cache
+
+          def compose!(deps = nil)
+            super(self)
+
+            inherit!(deps)
+          end
+
+          private
+
+          def inherit!(deps)
+            return unless deps
+
+            self.class.nodes.each do |key, factory|
+              next unless factory.inheritable?
+
+              root_entry = deps[key]
+              next unless root_entry.specified?
+
+              if self[key].specified?
+                raise DuplicateError, "#{key} is defined in top-level and `default:` entry"
+              end
+
+              @entries[key] = root_entry
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb
deleted file mode 100644
index 2b5a59c078e..00000000000
--- a/lib/gitlab/ci/config/entry/global.rb
+++ /dev/null
@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    class Config
-      module Entry
-        ##
-        # This class represents a global entry - root Entry for entire
-        # GitLab CI Configuration file.
-        #
-        class Global < ::Gitlab::Config::Entry::Node
-          include ::Gitlab::Config::Entry::Configurable
-
-          entry :before_script, Entry::Script,
-            description: 'Script that will be executed before each job.'
-
-          entry :image, Entry::Image,
-            description: 'Docker image that will be used to execute jobs.'
-
-          entry :include, Entry::Includes,
-                description: 'List of external YAML files to include.'
-
-          entry :services, Entry::Services,
-            description: 'Docker images that will be linked to the container.'
-
-          entry :after_script, Entry::Script,
-            description: 'Script that will be executed after each job.'
-
-          entry :variables, Entry::Variables,
-            description: 'Environment variables that will be used.'
-
-          entry :stages, Entry::Stages,
-            description: 'Configuration of stages for this pipeline.'
-
-          entry :types, Entry::Stages,
-            description: 'Deprecated: stages for this pipeline.'
-
-          entry :cache, Entry::Cache,
-            description: 'Configure caching between build jobs.'
-
-          helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache, :jobs
-
-          def compose!(_deps = nil)
-            super(self) do
-              compose_jobs!
-              compose_deprecated_entries!
-            end
-          end
-
-          private
-
-          # rubocop: disable CodeReuse/ActiveRecord
-          def compose_jobs!
-            factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs)
-              .value(@config.except(*self.class.nodes.keys))
-              .with(key: :jobs, parent: self,
-                    description: 'Jobs definition for this pipeline')
-
-            @entries[:jobs] = factory.create!
-          end
-          # rubocop: enable CodeReuse/ActiveRecord
-
-          def compose_deprecated_entries!
-            ##
-            # Deprecated `:types` key workaround - if types are defined and
-            # stages are not defined we use types definition as stages.
-            #
-            if types_defined? && !stages_defined?
-              @entries[:stages] = @entries[:types]
-            end
-
-            @entries.delete(:types)
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb
index 76e5d05639f..f9a9f036527 100644
--- a/lib/gitlab/ci/config/entry/hidden.rb
+++ b/lib/gitlab/ci/config/entry/hidden.rb
@@ -14,6 +14,14 @@ module Gitlab
             validates :config, presence: true
           end
 
+          def self.matching?(name, config)
+            name.to_s.start_with?('.')
+          end
+
+          def self.visible?
+            false
+          end
+
           def relevant?
             false
           end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 762532f7007..5ab795359b8 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -42,7 +42,8 @@ module Gitlab
           end
 
           entry :before_script, Entry::Script,
-            description: 'Global before script overridden in this job.'
+            description: 'Global before script overridden in this job.',
+            inherit: true
 
           entry :script, Entry::Commands,
             description: 'Commands that will be executed in this job.'
@@ -54,16 +55,20 @@ module Gitlab
             description: 'Deprecated: stage this job will be executed into.'
 
           entry :after_script, Entry::Script,
-            description: 'Commands that will be executed when finishing job.'
+            description: 'Commands that will be executed when finishing job.',
+            inherit: true
 
           entry :cache, Entry::Cache,
-            description: 'Cache definition for this job.'
+            description: 'Cache definition for this job.',
+            inherit: true
 
           entry :image, Entry::Image,
-            description: 'Image that will be used to execute this job.'
+            description: 'Image that will be used to execute this job.',
+            inherit: true
 
           entry :services, Entry::Services,
-            description: 'Services that will be used to execute this job.'
+            description: 'Services that will be used to execute this job.',
+            inherit: true
 
           entry :only, Entry::Policy,
             description: 'Refs policy this job will be executed for.',
@@ -95,6 +100,15 @@ module Gitlab
           attributes :script, :tags, :allow_failure, :when, :dependencies,
                      :retry, :parallel, :extends, :start_in
 
+          def self.matching?(name, config)
+            !name.to_s.start_with?('.') &&
+              config.is_a?(Hash) && config.key?(:script)
+          end
+
+          def self.visible?
+            true
+          end
+
           def compose!(deps = nil)
             super do
               if type_defined? && !stage_defined?
@@ -129,15 +143,19 @@ module Gitlab
 
           private
 
+          # We inherit config entries from `default:`
+          # if the entry has the `inherit: true` flag set
           def inherit!(deps)
             return unless deps
 
-            self.class.nodes.each_key do |key|
-              global_entry = deps[key]
+            self.class.nodes.each do |key, factory|
+              next unless factory.inheritable?
+
+              default_entry = deps.default[key]
               job_entry = self[key]
 
-              if global_entry.specified? && !job_entry.specified?
-                @entries[key] = global_entry
+              if default_entry.specified? && !job_entry.specified?
+                @entries[key] = default_entry
               end
             end
           end
@@ -152,7 +170,7 @@ module Gitlab
               cache: cache_value,
               only: only_value,
               except: except_value,
-              variables: variables_defined? ? variables_value : nil,
+              variables: variables_defined? ? variables_value : {},
               environment: environment_defined? ? environment_value : nil,
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index 9845c4af655..9d1a1ee8c4b 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -14,29 +14,48 @@ module Gitlab
             validates :config, type: Hash
 
             validate do
+              unless has_valid_jobs?
+                errors.add(:config, 'should contain valid jobs')
+              end
+
               unless has_visible_job?
                 errors.add(:config, 'should contain at least one visible job')
               end
             end
 
+            def has_valid_jobs?
+              config.all? do |name, value|
+                Jobs.find_type(name, value)
+              end
+            end
+
             def has_visible_job?
-              config.any? { |name, _| !hidden?(name) }
+              config.any? do |name, value|
+                Jobs.find_type(name, value)&.visible?
+              end
             end
           end
 
-          def hidden?(name)
-            name.to_s.start_with?('.')
+          TYPES = [Entry::Hidden, Entry::Job].freeze
+
+          private_constant :TYPES
+
+          def self.all_types
+            TYPES
           end
 
-          def node_type(name)
-            hidden?(name) ? Entry::Hidden : Entry::Job
+          def self.find_type(name, config)
+            self.all_types.find do |type|
+              type.matching?(name, config)
+            end
           end
 
           # rubocop: disable CodeReuse/ActiveRecord
           def compose!(deps = nil)
             super do
               @config.each do |name, config|
-                node = node_type(name)
+                node = self.class.find_type(name, config)
+                next unless node
 
                 factory = ::Gitlab::Config::Entry::Factory.new(node)
                   .value(config || {})
diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb
new file mode 100644
index 00000000000..0589ad3edf9
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/root.rb
@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # This class represents a global entry - root Entry for entire
+        # GitLab CI Configuration file.
+        #
+        class Root < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+
+          ALLOWED_KEYS = %i[default include before_script image services
+                            after_script variables stages types cache].freeze
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS
+          end
+
+          # reserved:
+          #   defines whether the node name is reserved
+          #   the reserved name cannot be used a job name
+          #   reserved should not be used as it will make
+          #   breaking change to `.gitlab-ci.yml`
+
+          entry :default, Entry::Default,
+            description: 'Default configuration for all jobs.',
+            default: {}
+
+          entry :include, Entry::Includes,
+            description: 'List of external YAML files to include.',
+            reserved: true
+
+          entry :before_script, Entry::Script,
+            description: 'Script that will be executed before each job.',
+            reserved: true
+
+          entry :image, Entry::Image,
+            description: 'Docker image that will be used to execute jobs.',
+            reserved: true
+
+          entry :services, Entry::Services,
+            description: 'Docker images that will be linked to the container.',
+            reserved: true
+
+          entry :after_script, Entry::Script,
+            description: 'Script that will be executed after each job.',
+            reserved: true
+
+          entry :variables, Entry::Variables,
+            description: 'Environment variables that will be used.',
+            reserved: true
+
+          entry :stages, Entry::Stages,
+            description: 'Configuration of stages for this pipeline.',
+            reserved: true
+
+          entry :types, Entry::Stages,
+            description: 'Deprecated: stages for this pipeline.',
+            reserved: true
+
+          entry :cache, Entry::Cache,
+            description: 'Configure caching between build jobs.',
+            reserved: true
+
+          helpers :default, :jobs, :stages, :types, :variables
+
+          delegate :before_script_value,
+                   :image_value,
+                   :services_value,
+                   :after_script_value,
+                   :cache_value, to: :default
+
+          attr_reader :jobs_config
+
+          class << self
+            include ::Gitlab::Utils::StrongMemoize
+
+            def reserved_nodes_names
+              strong_memoize(:reserved_nodes_names) do
+                self.nodes.select do |_, node|
+                  node.reserved?
+                end.keys
+              end
+            end
+          end
+
+          def initialize(config, **metadata)
+            super do
+              filter_jobs!
+            end
+          end
+
+          def compose!(_deps = nil)
+            super(self) do
+              compose_deprecated_entries!
+              compose_jobs!
+            end
+          end
+
+          def default
+            self[:default]
+          end
+
+          private
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def compose_jobs!
+            factory = ::Gitlab::Config::Entry::Factory.new(Entry::Jobs)
+              .value(jobs_config)
+              .with(key: :jobs, parent: self,
+                    description: 'Jobs definition for this pipeline')
+
+            @entries[:jobs] = factory.create!
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def compose_deprecated_entries!
+            ##
+            # Deprecated `:types` key workaround - if types are defined and
+            # stages are not defined we use types definition as stages.
+            #
+            if types_defined? && !stages_defined?
+              @entries[:stages] = @entries[:types]
+            end
+
+            @entries.delete(:types)
+          end
+
+          def filter_jobs!
+            return unless @config.is_a?(Hash)
+
+            @jobs_config = @config
+              .except(*self.class.reserved_nodes_names)  # rubocop: disable CodeReuse/ActiveRecord
+              .select do |name, config|
+              Entry::Jobs.find_type(name, config).present?
+            end
+
+            @config = @config.except(*@jobs_config.keys) # rubocop: disable CodeReuse/ActiveRecord
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 07ba6f83d47..a5693dc4f81 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -7,7 +7,7 @@ module Gitlab
 
       include Gitlab::Config::Entry::LegacyValidationHelpers
 
-      attr_reader :cache, :stages, :jobs
+      attr_reader :stages, :jobs
 
       def initialize(config, opts = {})
         @ci_config = Gitlab::Ci::Config.new(config, **opts)
@@ -95,13 +95,8 @@ module Gitlab
         ##
         # Global config
         #
-        @before_script = @ci_config.before_script
-        @image = @ci_config.image
-        @after_script = @ci_config.after_script
-        @services = @ci_config.services
         @variables = @ci_config.variables
         @stages = @ci_config.stages
-        @cache = @ci_config.cache
 
         ##
         # Jobs
diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb
index 6667a5d3d33..b7ec4b7c4f8 100644
--- a/lib/gitlab/config/entry/configurable.rb
+++ b/lib/gitlab/config/entry/configurable.rb
@@ -58,13 +58,21 @@ module Gitlab
             Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }]
           end
 
+          def reserved_node_names
+            self.nodes.select do |_, node|
+              node.reserved?
+            end.keys
+          end
+
           private
 
           # rubocop: disable CodeReuse/ActiveRecord
-          def entry(key, entry, metadata)
+          def entry(key, entry, description: nil, default: nil, inherit: nil, reserved: nil)
             factory = ::Gitlab::Config::Entry::Factory.new(entry)
-              .with(description: metadata[:description])
-              .with(default: metadata[:default])
+              .with(description: description)
+              .with(default: default)
+              .with(inherit: inherit)
+              .with(reserved: reserved)
 
             (@nodes ||= {}).merge!(key.to_sym => factory)
           end
diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb
index 3c06b1e0d24..8f1f4a81bb5 100644
--- a/lib/gitlab/config/entry/factory.rb
+++ b/lib/gitlab/config/entry/factory.rb
@@ -30,6 +30,18 @@ module Gitlab
           self
         end
 
+        def description
+          @attributes[:description]
+        end
+
+        def inheritable?
+          @attributes[:inherit]
+        end
+
+        def reserved?
+          @attributes[:reserved]
+        end
+
         def create!
           raise InvalidFactory unless defined?(@value)