Merge remote-tracking branch 'upstream/master' into feature/1376-allow-write-access-deploy-keys

* upstream/master: (3852 commits)
  Grapify token API
  Fix cache for commit status in commits list to respect branches
  Grapify milestones API
  Grapify runners API
  Improve EeCompatCheck, cache EE repo and keep artifacts for the ee_compat_check task
  Use 'Forking in progress' title when appropriate
  Fix CHANGELOG after 8.14.0-rc1 tag
  Update CHANGELOG.md for 8.14.0-rc1
  Fix YAML syntax on CHANGELOG entry
  Remove redundant rescue from repository keep_around
  Remove redundant space from repository model code
  Remove order-dependent expectation
  Minor CHANGELOG.md cleanups
  Add a link to Git cheatsheet PDF in docs readme
  Grapify the session API
  Add 8.13.5, 8.12.9, and 8.11.11 CHANGELOG
  Merge branch 'unauthenticated-container-registry-access' into 'security'
  Merge branch '23403-fix-events-for-private-project-features' into 'security'
  Merge branch 'fix-unathorized-cloning' into 'security'
  Merge branch 'markdown-xss-fix-option-2.1' into 'security'
  ...

11 files changed, 419 insertions, 109 deletions

diff --git a/spec/lib/gitlab/ci/config/node/cache_spec.rb b/spec/lib/gitlab/ci/config/node/cache_spec.rb
index 50f619ce26e..e251210949c 100644
--- a/spec/lib/gitlab/ci/config/node/cache_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/cache_spec.rb
@@ -4,7 +4,7 @@ describe Gitlab::Ci::Config::Node::Cache do
   let(:entry) { described_class.new(config) }
 
   describe 'validations' do
-    before { entry.process! }
+    before { entry.compose! }
 
     context 'when entry config value is correct' do
       let(:config) do
diff --git a/spec/lib/gitlab/ci/config/node/environment_spec.rb b/spec/lib/gitlab/ci/config/node/environment_spec.rb
new file mode 100644
index 00000000000..df925ff1afd
--- /dev/null
+++ b/spec/lib/gitlab/ci/config/node/environment_spec.rb
@@ -0,0 +1,217 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Environment do
+  let(:entry) { described_class.new(config) }
+
+  before { entry.compose! }
+
+  context 'when configuration is a string' do
+    let(:config) { 'production' }
+
+    describe '#string?' do
+      it 'is string configuration' do
+        expect(entry).to be_string
+      end
+    end
+
+    describe '#hash?' do
+      it 'is not hash configuration' do
+        expect(entry).not_to be_hash
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+
+    describe '#value' do
+      it 'returns valid hash' do
+        expect(entry.value).to include(name: 'production')
+      end
+    end
+
+    describe '#name' do
+      it 'returns environment name' do
+        expect(entry.name).to eq 'production'
+      end
+    end
+
+    describe '#url' do
+      it 'returns environment url' do
+        expect(entry.url).to be_nil
+      end
+    end
+  end
+
+  context 'when configuration is a hash' do
+    let(:config) do
+      { name: 'development', url: 'https://example.gitlab.com' }
+    end
+
+    describe '#string?' do
+      it 'is not string configuration' do
+        expect(entry).not_to be_string
+      end
+    end
+
+    describe '#hash?' do
+      it 'is hash configuration' do
+        expect(entry).to be_hash
+      end
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+
+    describe '#value' do
+      it 'returns valid hash' do
+        expect(entry.value).to eq config
+      end
+    end
+
+    describe '#name' do
+      it 'returns environment name' do
+        expect(entry.name).to eq 'development'
+      end
+    end
+
+    describe '#url' do
+      it 'returns environment url' do
+        expect(entry.url).to eq 'https://example.gitlab.com'
+      end
+    end
+  end
+
+  context 'when valid action is used' do
+    let(:config) do
+      { name: 'production',
+        action: 'start' }
+    end
+
+    it 'is valid' do
+      expect(entry).to be_valid
+    end
+  end
+
+  context 'when invalid action is used' do
+    let(:config) do
+      { name: 'production',
+        action: 'invalid' }
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
+      end
+    end
+
+    describe '#errors' do
+      it 'contains error about invalid action' do
+        expect(entry.errors)
+          .to include 'environment action should be start or stop'
+      end
+    end
+  end
+
+  context 'when on_stop is used' do
+    let(:config) do
+      { name: 'production',
+        on_stop: 'close_app' }
+    end
+
+    it 'is valid' do
+      expect(entry).to be_valid
+    end
+  end
+
+  context 'when invalid on_stop is used' do
+    let(:config) do
+      { name: 'production',
+        on_stop: false }
+    end
+
+    describe '#valid?' do
+      it 'is not valid' do
+        expect(entry).not_to be_valid
+      end
+    end
+
+    describe '#errors' do
+      it 'contains error about invalid action' do
+        expect(entry.errors)
+          .to include 'environment on stop should be a string'
+      end
+    end
+  end
+
+  context 'when variables are used for environment' do
+    let(:config) do
+      { name: 'review/$CI_BUILD_REF_NAME',
+        url: 'https://$CI_BUILD_REF_NAME.review.gitlab.com' }
+    end
+
+    describe '#valid?' do
+      it 'is valid' do
+        expect(entry).to be_valid
+      end
+    end
+  end
+
+  context 'when configuration is invalid' do
+    context 'when configuration is an array' do
+      let(:config) { ['env'] }
+
+      describe '#valid?' do
+        it 'is not valid' do
+          expect(entry).not_to be_valid
+        end
+      end
+
+      describe '#errors' do
+        it 'contains error about invalid type' do
+          expect(entry.errors)
+            .to include 'environment config should be a hash or a string'
+        end
+      end
+    end
+
+    context 'when environment name is not present' do
+      let(:config) { { url: 'https://example.gitlab.com' } }
+
+      describe '#valid?' do
+        it 'is not valid' do
+          expect(entry).not_to be_valid
+        end
+      end
+
+      describe '#errors?' do
+        it 'contains error about missing environment name' do
+          expect(entry.errors)
+            .to include "environment name can't be blank"
+        end
+      end
+    end
+
+    context 'when invalid URL is used' do
+      let(:config) { { name: 'test', url: 'invalid-example.gitlab.com' } }
+
+      describe '#valid?' do
+        it 'is not valid' do
+          expect(entry).not_to be_valid
+        end
+      end
+
+      describe '#errors?' do
+        it 'contains error about invalid URL' do
+          expect(entry.errors)
+            .to include "environment url must be a valid url"
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/config/node/factory_spec.rb b/spec/lib/gitlab/ci/config/node/factory_spec.rb
index d26185ba585..a699089c563 100644
--- a/spec/lib/gitlab/ci/config/node/factory_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/factory_spec.rb
@@ -65,7 +65,8 @@ describe Gitlab::Ci::Config::Node::Factory do
           .value(nil)
           .create!
 
-        expect(entry).to be_an_instance_of Gitlab::Ci::Config::Node::Undefined
+        expect(entry)
+          .to be_an_instance_of Gitlab::Ci::Config::Node::Unspecified
       end
     end
 
diff --git a/spec/lib/gitlab/ci/config/node/global_spec.rb b/spec/lib/gitlab/ci/config/node/global_spec.rb
index 2f87d270b36..12232ff7e2f 100644
--- a/spec/lib/gitlab/ci/config/node/global_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/global_spec.rb
@@ -14,7 +14,7 @@ describe Gitlab::Ci::Config::Node::Global do
   end
 
   context 'when hash is valid' do
-    context 'when all entries defined' do
+    context 'when some entries defined' do
       let(:hash) do
         { before_script: ['ls', 'pwd'],
           image: 'ruby:2.2',
@@ -24,11 +24,11 @@ describe Gitlab::Ci::Config::Node::Global do
           stages: ['build', 'pages'],
           cache: { key: 'k', untracked: true, paths: ['public/'] },
           rspec: { script: %w[rspec ls] },
-          spinach: { script: 'spinach' } }
+          spinach: { before_script: [], variables: {}, script: 'spinach' } }
       end
 
-      describe '#process!' do
-        before { global.process! }
+      describe '#compose!' do
+        before { global.compose! }
 
         it 'creates nodes hash' do
           expect(global.descendants).to be_an Array
@@ -59,7 +59,7 @@ describe Gitlab::Ci::Config::Node::Global do
         end
       end
 
-      context 'when not processed' do
+      context 'when not composed' do
         describe '#before_script' do
           it 'returns nil' do
             expect(global.before_script).to be nil
@@ -73,8 +73,14 @@ describe Gitlab::Ci::Config::Node::Global do
         end
       end
 
-      context 'when processed' do
-        before { global.process! }
+      context 'when composed' do
+        before { global.compose! }
+
+        describe '#errors' do
+          it 'has no errors' do
+            expect(global.errors).to be_empty
+          end
+        end
 
         describe '#before_script' do
           it 'returns correct script' do
@@ -137,10 +143,24 @@ describe Gitlab::Ci::Config::Node::Global do
             expect(global.jobs).to eq(
               rspec: { name: :rspec,
                        script: %w[rspec ls],
-                       stage: 'test' },
+                       before_script: ['ls', 'pwd'],
+                       commands: "ls\npwd\nrspec\nls",
+                       image: 'ruby:2.2',
+                       services: ['postgres:9.1', 'mysql:5.5'],
+                       stage: 'test',
+                       cache: { key: 'k', untracked: true, paths: ['public/'] },
+                       variables: { VAR: 'value' },
+                       after_script: ['make clean'] },
               spinach: { name: :spinach,
+                         before_script: [],
                          script: %w[spinach],
-                         stage: 'test' }
+                         commands: 'spinach',
+                         image: 'ruby:2.2',
+                         services: ['postgres:9.1', 'mysql:5.5'],
+                         stage: 'test',
+                         cache: { key: 'k', untracked: true, paths: ['public/'] },
+                         variables: {},
+                         after_script: ['make clean'] },
             )
           end
         end
@@ -148,17 +168,20 @@ describe Gitlab::Ci::Config::Node::Global do
     end
 
     context 'when most of entires not defined' do
-      let(:hash) { { cache: { key: 'a' }, rspec: { script: %w[ls] } } }
-      before { global.process! }
+      before { global.compose! }
+
+      let(:hash) do
+        { cache: { key: 'a' }, rspec: { script: %w[ls] } }
+      end
 
       describe '#nodes' do
         it 'instantizes all nodes' do
           expect(global.descendants.count).to eq 8
         end
 
-        it 'contains undefined nodes' do
+        it 'contains unspecified nodes' do
           expect(global.descendants.first)
-            .to be_an_instance_of Gitlab::Ci::Config::Node::Undefined
+            .to be_an_instance_of Gitlab::Ci::Config::Node::Unspecified
         end
       end
 
@@ -188,8 +211,11 @@ describe Gitlab::Ci::Config::Node::Global do
     # details.
     #
     context 'when entires specified but not defined' do
-      let(:hash) { { variables: nil, rspec: { script: 'rspec' } } }
-      before { global.process! }
+      before { global.compose! }
+
+      let(:hash) do
+        { variables: nil, rspec: { script: 'rspec' } }
+      end
 
       describe '#variables' do
         it 'undefined entry returns a default value' do
@@ -200,7 +226,7 @@ describe Gitlab::Ci::Config::Node::Global do
   end
 
   context 'when hash is not valid' do
-    before { global.process! }
+    before { global.compose! }
 
     let(:hash) do
       { before_script: 'ls' }
@@ -247,4 +273,27 @@ describe Gitlab::Ci::Config::Node::Global do
       expect(global.specified?).to be true
     end
   end
+
+  describe '#[]' do
+    before { global.compose! }
+
+    let(:hash) do
+      { cache: { key: 'a' }, rspec: { script: 'ls' } }
+    end
+
+    context 'when node exists' do
+      it 'returns correct entry' do
+        expect(global[:cache])
+          .to be_an_instance_of Gitlab::Ci::Config::Node::Cache
+        expect(global[:jobs][:rspec][:script].value).to eq ['ls']
+      end
+    end
+
+    context 'when node does not exist' do
+      it 'always return unspecified node' do
+        expect(global[:some][:unknown][:node])
+          .not_to be_specified
+      end
+    end
+  end
 end
diff --git a/spec/lib/gitlab/ci/config/node/hidden_job_spec.rb b/spec/lib/gitlab/ci/config/node/hidden_spec.rb
index cc44e2cc054..61e2a554419 100644
--- a/spec/lib/gitlab/ci/config/node/hidden_job_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/hidden_spec.rb
@@ -1,15 +1,15 @@
 require 'spec_helper'
 
-describe Gitlab::Ci::Config::Node::HiddenJob do
+describe Gitlab::Ci::Config::Node::Hidden do
   let(:entry) { described_class.new(config) }
 
   describe 'validations' do
     context 'when entry config value is correct' do
-      let(:config) { { image: 'ruby:2.2' } }
+      let(:config) { [:some, :array] }
 
       describe '#value' do
         it 'returns key value' do
-          expect(entry.value).to eq(image: 'ruby:2.2')
+          expect(entry.value).to eq [:some, :array]
         end
       end
 
@@ -21,17 +21,6 @@ describe Gitlab::Ci::Config::Node::HiddenJob do
     end
 
     context 'when entry value is not correct' do
-      context 'incorrect config value type' do
-        let(:config) { ['incorrect'] }
-
-        describe '#errors' do
-          it 'saves errors' do
-            expect(entry.errors)
-              .to include 'hidden job config should be a hash'
-          end
-        end
-      end
-
       context 'when config is empty' do
         let(:config) { {} }
 
diff --git a/spec/lib/gitlab/ci/config/node/job_spec.rb b/spec/lib/gitlab/ci/config/node/job_spec.rb
index 1484fb60dd8..91f676dae03 100644
--- a/spec/lib/gitlab/ci/config/node/job_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/job_spec.rb
@@ -3,9 +3,9 @@ require 'spec_helper'
 describe Gitlab::Ci::Config::Node::Job do
   let(:entry) { described_class.new(config, name: :rspec) }
 
-  before { entry.process! }
-
   describe 'validations' do
+    before { entry.compose! }
+
     context 'when entry config value is correct' do
       let(:config) { { script: 'rspec' } }
 
@@ -59,28 +59,82 @@ describe Gitlab::Ci::Config::Node::Job do
     end
   end
 
-  describe '#value' do
-    context 'when entry is correct' do
+  describe '#relevant?' do
+    it 'is a relevant entry' do
+      expect(entry).to be_relevant
+    end
+  end
+
+  describe '#compose!' do
+    let(:unspecified) { double('unspecified', 'specified?' => false) }
+
+    let(:specified) do
+      double('specified', 'specified?' => true, value: 'specified')
+    end
+
+    let(:deps) { double('deps', '[]' => unspecified) }
+
+    context 'when job config overrides global config' do
+      before { entry.compose!(deps) }
+
       let(:config) do
-        { before_script: %w[ls pwd],
-          script: 'rspec',
-          after_script: %w[cleanup] }
+        { image: 'some_image', cache: { key: 'test' } }
+      end
+
+      it 'overrides global config' do
+        expect(entry[:image].value).to eq 'some_image'
+        expect(entry[:cache].value).to eq(key: 'test')
+      end
+    end
+
+    context 'when job config does not override global config' do
+      before do
+        allow(deps).to receive('[]').with(:image).and_return(specified)
+        entry.compose!(deps)
       end
 
-      it 'returns correct value' do
-        expect(entry.value)
-          .to eq(name: :rspec,
-                 before_script: %w[ls pwd],
-                 script: %w[rspec],
-                 stage: 'test',
-                 after_script: %w[cleanup])
+      let(:config) { { script: 'ls', cache: { key: 'test' } } }
+
+      it 'uses config from global entry' do
+        expect(entry[:image].value).to eq 'specified'
+        expect(entry[:cache].value).to eq(key: 'test')
       end
     end
   end
 
-  describe '#relevant?' do
-    it 'is a relevant entry' do
-      expect(entry).to be_relevant
+  context 'when composed' do
+    before { entry.compose! }
+
+    describe '#value' do
+      before { entry.compose! }
+
+      context 'when entry is correct' do
+        let(:config) do
+          { before_script: %w[ls pwd],
+            script: 'rspec',
+            after_script: %w[cleanup] }
+        end
+
+        it 'returns correct value' do
+          expect(entry.value)
+            .to eq(name: :rspec,
+                   before_script: %w[ls pwd],
+                   script: %w[rspec],
+                   commands: "ls\npwd\nrspec",
+                   stage: 'test',
+                   after_script: %w[cleanup])
+        end
+      end
+    end
+
+    describe '#commands' do
+      let(:config) do
+        { before_script: %w[ls pwd], script: 'rspec' }
+      end
+
+      it 'returns a string of commands concatenated with new line character' do
+        expect(entry.commands).to eq "ls\npwd\nrspec"
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/ci/config/node/jobs_spec.rb b/spec/lib/gitlab/ci/config/node/jobs_spec.rb
index b8d9c70479c..929809339ef 100644
--- a/spec/lib/gitlab/ci/config/node/jobs_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/jobs_spec.rb
@@ -4,7 +4,7 @@ describe Gitlab::Ci::Config::Node::Jobs do
   let(:entry) { described_class.new(config) }
 
   describe 'validations' do
-    before { entry.process! }
+    before { entry.compose! }
 
     context 'when entry config value is correct' do
       let(:config) { { rspec: { script: 'rspec' } } }
@@ -47,8 +47,8 @@ describe Gitlab::Ci::Config::Node::Jobs do
     end
   end
 
-  context 'when valid job entries processed' do
-    before { entry.process! }
+  context 'when valid job entries composed' do
+    before { entry.compose! }
 
     let(:config) do
       { rspec: { script: 'rspec' },
@@ -61,9 +61,11 @@ describe Gitlab::Ci::Config::Node::Jobs do
         expect(entry.value).to eq(
           rspec: { name: :rspec,
                    script: %w[rspec],
+                   commands: 'rspec',
                    stage: 'test' },
           spinach: { name: :spinach,
                      script: %w[spinach],
+                     commands: 'spinach',
                      stage: 'test' })
       end
     end
@@ -74,7 +76,7 @@ describe Gitlab::Ci::Config::Node::Jobs do
         expect(entry.descendants.first(2))
           .to all(be_an_instance_of(Gitlab::Ci::Config::Node::Job))
         expect(entry.descendants.last)
-          .to be_an_instance_of(Gitlab::Ci::Config::Node::HiddenJob)
+          .to be_an_instance_of(Gitlab::Ci::Config::Node::Hidden)
       end
     end
 
diff --git a/spec/lib/gitlab/ci/config/node/null_spec.rb b/spec/lib/gitlab/ci/config/node/null_spec.rb
deleted file mode 100644
index 1ab5478dcfa..00000000000
--- a/spec/lib/gitlab/ci/config/node/null_spec.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-require 'spec_helper'
-
-describe Gitlab::Ci::Config::Node::Null do
-  let(:null) { described_class.new(nil) }
-
-  describe '#leaf?' do
-    it 'is leaf node' do
-      expect(null).to be_leaf
-    end
-  end
-
-  describe '#valid?' do
-    it 'is always valid' do
-      expect(null).to be_valid
-    end
-  end
-
-  describe '#errors' do
-    it 'is does not contain errors' do
-      expect(null.errors).to be_empty
-    end
-  end
-
-  describe '#value' do
-    it 'returns nil' do
-      expect(null.value).to eq nil
-    end
-  end
-
-  describe '#relevant?' do
-    it 'is not relevant' do
-      expect(null.relevant?).to eq false
-    end
-  end
-
-  describe '#specified?' do
-    it 'is not defined' do
-      expect(null.specified?).to eq false
-    end
-  end
-end
diff --git a/spec/lib/gitlab/ci/config/node/script_spec.rb b/spec/lib/gitlab/ci/config/node/script_spec.rb
index ee7395362a9..219a7e981d3 100644
--- a/spec/lib/gitlab/ci/config/node/script_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/script_spec.rb
@@ -3,9 +3,7 @@ require 'spec_helper'
 describe Gitlab::Ci::Config::Node::Script do
   let(:entry) { described_class.new(config) }
 
-  describe '#process!' do
-    before { entry.process! }
-
+  describe 'validations' do
     context 'when entry config value is correct' do
       let(:config) { ['ls', 'pwd'] }
 
diff --git a/spec/lib/gitlab/ci/config/node/undefined_spec.rb b/spec/lib/gitlab/ci/config/node/undefined_spec.rb
index 2d43e1c1a9d..6bde8602963 100644
--- a/spec/lib/gitlab/ci/config/node/undefined_spec.rb
+++ b/spec/lib/gitlab/ci/config/node/undefined_spec.rb
@@ -1,32 +1,41 @@
 require 'spec_helper'
 
 describe Gitlab::Ci::Config::Node::Undefined do
-  let(:undefined) { described_class.new(entry) }
-  let(:entry) { spy('Entry') }
+  let(:entry) { described_class.new }
+
+  describe '#leaf?' do
+    it 'is leaf node' do
+      expect(entry).to be_leaf
+    end
+  end
 
   describe '#valid?' do
-    it 'delegates method to entry' do
-      expect(undefined.valid).to eq entry
+    it 'is always valid' do
+      expect(entry).to be_valid
     end
   end
 
   describe '#errors' do
-    it 'delegates method to entry' do
-      expect(undefined.errors).to eq entry
+    it 'is does not contain errors' do
+      expect(entry.errors).to be_empty
     end
   end
 
   describe '#value' do
-    it 'delegates method to entry' do
-      expect(undefined.value).to eq entry
+    it 'returns nil' do
+      expect(entry.value).to eq nil
     end
   end
 
-  describe '#specified?' do
-    it 'is always false' do
-      allow(entry).to receive(:specified?).and_return(true)
+  describe '#relevant?' do
+    it 'is not relevant' do
+      expect(entry.relevant?).to eq false
+    end
+  end
 
-      expect(undefined.specified?).to be false
+  describe '#specified?' do
+    it 'is not defined' do
+      expect(entry.specified?).to eq false
     end
   end
 end
diff --git a/spec/lib/gitlab/ci/config/node/unspecified_spec.rb b/spec/lib/gitlab/ci/config/node/unspecified_spec.rb
new file mode 100644
index 00000000000..ba3ceef24ce
--- /dev/null
+++ b/spec/lib/gitlab/ci/config/node/unspecified_spec.rb
@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Node::Unspecified do
+  let(:unspecified) { described_class.new(entry) }
+  let(:entry) { spy('Entry') }
+
+  describe '#valid?' do
+    it 'delegates method to entry' do
+      expect(unspecified.valid?).to eq entry
+    end
+  end
+
+  describe '#errors' do
+    it 'delegates method to entry' do
+      expect(unspecified.errors).to eq entry
+    end
+  end
+
+  describe '#value' do
+    it 'delegates method to entry' do
+      expect(unspecified.value).to eq entry
+    end
+  end
+
+  describe '#specified?' do
+    it 'is always false' do
+      allow(entry).to receive(:specified?).and_return(true)
+
+      expect(unspecified.specified?).to be false
+    end
+  end
+end