Merge branch 'qa/gb/ci-cd-qa-pipelines' into 'master'

CI/CD end-to-end tests

Closes gitlab-qa#18

See merge request gitlab-org/gitlab-ce!16619

23 files changed, 428 insertions, 93 deletions

diff --git a/app/views/ci/runner/_how_to_setup_runner.html.haml b/app/views/ci/runner/_how_to_setup_runner.html.haml
index b75dab0acc5..8db7727b80c 100644
--- a/app/views/ci/runner/_how_to_setup_runner.html.haml
+++ b/app/views/ci/runner/_how_to_setup_runner.html.haml
@@ -8,7 +8,7 @@
       = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe
     %li
       = _("Specify the following URL during the Runner setup:")
-      %code= root_url(only_path: false)
+      %code#coordinator_address= root_url(only_path: false)
     %li
       = _("Use the following registration token during setup:")
       %code#registration_token= registration_token
diff --git a/qa/qa.rb b/qa/qa.rb
index fa2cabe0e46..ac51bb445f2 100644
--- a/qa/qa.rb
+++ b/qa/qa.rb
@@ -28,6 +28,7 @@ module QA
       autoload :Group, 'qa/factory/resource/group'
       autoload :Project, 'qa/factory/resource/project'
       autoload :DeployKey, 'qa/factory/resource/deploy_key'
+      autoload :Runner, 'qa/factory/resource/runner'
       autoload :PersonalAccessToken, 'qa/factory/resource/personal_access_token'
     end
 
@@ -49,7 +50,7 @@ module QA
     #
     autoload :Bootable, 'qa/scenario/bootable'
     autoload :Actable, 'qa/scenario/actable'
-    autoload :Entrypoint, 'qa/scenario/entrypoint'
+    autoload :Taggable, 'qa/scenario/taggable'
     autoload :Template, 'qa/scenario/template'
 
     ##
@@ -108,7 +109,14 @@ module QA
       module Settings
         autoload :Common, 'qa/page/project/settings/common'
         autoload :Repository, 'qa/page/project/settings/repository'
+        autoload :CICD, 'qa/page/project/settings/ci_cd'
         autoload :DeployKeys, 'qa/page/project/settings/deploy_keys'
+        autoload :Runners, 'qa/page/project/settings/runners'
+      end
+
+      module Pipeline
+        autoload :Index, 'qa/page/project/pipeline/index'
+        autoload :Show, 'qa/page/project/pipeline/show'
       end
     end
 
@@ -134,10 +142,13 @@ module QA
   end
 
   ##
-  # Classes describing shell interaction with GitLab
+  # Classes describing services being part of GitLab and how we can interact
+  # with these services, like through the shell.
   #
-  module Shell
-    autoload :Omnibus, 'qa/shell/omnibus'
+  module Service
+    autoload :Shellout, 'qa/service/shellout'
+    autoload :Omnibus, 'qa/service/omnibus'
+    autoload :Runner, 'qa/service/runner'
   end
 
   ##
diff --git a/qa/qa/factory/resource/deploy_key.rb b/qa/qa/factory/resource/deploy_key.rb
index 671114d38b1..25d2af6e321 100644
--- a/qa/qa/factory/resource/deploy_key.rb
+++ b/qa/qa/factory/resource/deploy_key.rb
@@ -19,7 +19,7 @@ module QA
           project.visit!
 
           Page::Menu::Side.act do
-            click_repository_setting
+            click_repository_settings
           end
 
           Page::Project::Settings::Repository.perform do |setting|
diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb
new file mode 100644
index 00000000000..5f37f8ac2e9
--- /dev/null
+++ b/qa/qa/factory/resource/runner.rb
@@ -0,0 +1,42 @@
+require 'securerandom'
+
+module QA
+  module Factory
+    module Resource
+      class Runner < Factory::Base
+        attr_writer :name, :tags
+
+        dependency Factory::Resource::Project, as: :project do |project|
+          project.name = 'project-with-ci-cd'
+          project.description = 'Project with CI/CD Pipelines'
+        end
+
+        def name
+          @name || "qa-runner-#{SecureRandom.hex(4)}"
+        end
+
+        def tags
+          @tags || %w[qa e2e]
+        end
+
+        def fabricate!
+          project.visit!
+
+          Page::Menu::Side.act { click_ci_cd_settings }
+
+          Service::Runner.new(name).tap do |runner|
+            Page::Project::Settings::CICD.perform do |settings|
+              settings.expand_runners_settings do |runners|
+                runner.pull
+                runner.token = runners.registration_token
+                runner.address = runners.coordinator_address
+                runner.tags = tags
+                runner.register!
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/page/menu/side.rb b/qa/qa/page/menu/side.rb
index 1df4e0c2429..7f0f924c5e8 100644
--- a/qa/qa/page/menu/side.rb
+++ b/qa/qa/page/menu/side.rb
@@ -5,18 +5,35 @@ module QA
         view 'app/views/layouts/nav/sidebar/_project.html.haml' do
           element :settings_item
           element :repository_link, "title: 'Repository'"
+          element :pipelines_settings_link, "title: 'CI / CD'"
           element :top_level_items, '.sidebar-top-level-items'
         end
 
-        def click_repository_setting
-          hover_setting do
-            click_link('Repository')
+        def click_repository_settings
+          hover_settings do
+            within_submenu do
+              click_link('Repository')
+            end
+          end
+        end
+
+        def click_ci_cd_settings
+          hover_settings do
+            within_submenu do
+              click_link('CI / CD')
+            end
+          end
+        end
+
+        def click_ci_cd_pipelines
+          within_sidebar do
+            click_link('CI / CD')
           end
         end
 
         private
 
-        def hover_setting
+        def hover_settings
           within_sidebar do
             find('.qa-settings-item').hover
 
@@ -29,6 +46,12 @@ module QA
             yield
           end
         end
+
+        def within_submenu
+          page.within('.fly-out-list') do
+            yield
+          end
+        end
       end
     end
   end
diff --git a/qa/qa/page/project/pipeline/index.rb b/qa/qa/page/project/pipeline/index.rb
new file mode 100644
index 00000000000..32c108393b9
--- /dev/null
+++ b/qa/qa/page/project/pipeline/index.rb
@@ -0,0 +1,13 @@
+module QA::Page
+  module Project::Pipeline
+    class Index < QA::Page::Base
+      view 'app/assets/javascripts/pipelines/components/pipeline_url.vue' do
+        element :pipeline_link, 'class="js-pipeline-url-link"'
+      end
+
+      def go_to_latest_pipeline
+        first('.js-pipeline-url-link').click
+      end
+    end
+  end
+end
diff --git a/qa/qa/page/project/pipeline/show.rb b/qa/qa/page/project/pipeline/show.rb
new file mode 100644
index 00000000000..0835173f1cd
--- /dev/null
+++ b/qa/qa/page/project/pipeline/show.rb
@@ -0,0 +1,35 @@
+module QA::Page
+  module Project::Pipeline
+    class Show < QA::Page::Base
+      view 'app/assets/javascripts/vue_shared/components/header_ci_component.vue' do
+        element :pipeline_header, /header class.*ci-header-container.*/
+      end
+
+      view 'app/assets/javascripts/pipelines/components/graph/graph_component.vue' do
+        element :pipeline_graph, /class.*pipeline-graph.*/
+      end
+
+      view 'app/assets/javascripts/pipelines/components/graph/job_component.vue' do
+        element :job_component, /class.*ci-job-component.*/
+      end
+
+      view 'app/assets/javascripts/vue_shared/components/ci_icon.vue' do
+        element :status_icon, 'ci-status-icon-${status}'
+      end
+
+      def running?
+        within('.ci-header-container') do
+          return page.has_content?('running')
+        end
+      end
+
+      def has_build?(name, status: :success)
+        within('.pipeline-graph') do
+          within('.ci-job-component', text: name) do
+            return has_selector?(".ci-status-icon-#{status}")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/page/project/settings/ci_cd.rb b/qa/qa/page/project/settings/ci_cd.rb
new file mode 100644
index 00000000000..5270dde7411
--- /dev/null
+++ b/qa/qa/page/project/settings/ci_cd.rb
@@ -0,0 +1,21 @@
+module QA
+  module Page
+    module Project
+      module Settings
+        class CICD < Page::Base
+          include Common
+
+          view 'app/views/projects/settings/ci_cd/show.html.haml' do
+            element :runners_settings, 'Runners settings'
+          end
+
+          def expand_runners_settings(&block)
+            expand_section('Runners settings') do
+              Settings::Runners.perform(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/page/project/settings/common.rb b/qa/qa/page/project/settings/common.rb
index b4ef07e1540..1357bf031d5 100644
--- a/qa/qa/page/project/settings/common.rb
+++ b/qa/qa/page/project/settings/common.rb
@@ -10,6 +10,16 @@ module QA
               yield
             end
           end
+
+          def expand_section(name)
+            page.within('#content-body') do
+              page.within('section', text: name) do
+                click_button 'Expand'
+
+                yield
+              end
+            end
+          end
         end
       end
     end
diff --git a/qa/qa/page/project/settings/runners.rb b/qa/qa/page/project/settings/runners.rb
new file mode 100644
index 00000000000..b41668c94cd
--- /dev/null
+++ b/qa/qa/page/project/settings/runners.rb
@@ -0,0 +1,35 @@
+module QA
+  module Page
+    module Project
+      module Settings
+        class Runners < Page::Base
+          view 'app/views/ci/runner/_how_to_setup_runner.html.haml' do
+            element :registration_token, '%code#registration_token'
+            element :coordinator_address, '%code#coordinator_address'
+          end
+
+          ##
+          # TODO, phase-out CSS classes added in Ruby helpers.
+          #
+          view 'app/helpers/runners_helper.rb' do
+            # rubocop:disable Lint/InterpolationCheck
+            element :runner_status, 'runner-status-#{status}'
+            # rubocop:enable Lint/InterpolationCheck
+          end
+
+          def registration_token
+            find('code#registration_token').text
+          end
+
+          def coordinator_address
+            find('code#coordinator_address').text
+          end
+
+          def has_online_runner?
+            page.has_css?('.runner-status-online')
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb
index c8af5ba6280..5e66e40a0b5 100644
--- a/qa/qa/page/project/show.rb
+++ b/qa/qa/page/project/show.rb
@@ -33,6 +33,7 @@ module QA
 
         def wait_for_push
           sleep 5
+          refresh
         end
       end
     end
diff --git a/qa/qa/scenario/entrypoint.rb b/qa/qa/scenario/entrypoint.rb
deleted file mode 100644
index ae099fd911e..00000000000
--- a/qa/qa/scenario/entrypoint.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-module QA
-  module Scenario
-    ##
-    # Base class for running the suite against any GitLab instance,
-    # including staging and on-premises installation.
-    #
-    class Entrypoint < Template
-      include Bootable
-
-      def perform(address, *files)
-        Runtime::Scenario.define(:gitlab_address, address)
-
-        ##
-        # Perform before hooks, which are different for CE and EE
-        #
-        Runtime::Release.perform_before_hooks
-
-        Specs::Runner.perform do |specs|
-          specs.tty = true
-          specs.tags = self.class.get_tags
-          specs.files = files.any? ? files : 'qa/specs/features'
-        end
-      end
-
-      def self.tags(*tags)
-        @tags = tags
-      end
-
-      def self.get_tags
-        @tags
-      end
-    end
-  end
-end
diff --git a/qa/qa/scenario/taggable.rb b/qa/qa/scenario/taggable.rb
new file mode 100644
index 00000000000..b1f24d742e0
--- /dev/null
+++ b/qa/qa/scenario/taggable.rb
@@ -0,0 +1,17 @@
+module QA
+  module Scenario
+    module Taggable
+      # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
+      def tags(*tags)
+        @tags = tags
+      end
+
+      def focus
+        @tags.to_a
+      end
+
+      # rubocop:enable Gitlab/ModuleWithInstanceVariables
+    end
+  end
+end
diff --git a/qa/qa/scenario/test/instance.rb b/qa/qa/scenario/test/instance.rb
index e2a1f6bf2bd..993bbd723a3 100644
--- a/qa/qa/scenario/test/instance.rb
+++ b/qa/qa/scenario/test/instance.rb
@@ -2,11 +2,29 @@ module QA
   module Scenario
     module Test
       ##
-      # Run test suite against any GitLab instance,
+      # Base class for running the suite against any GitLab instance,
       # including staging and on-premises installation.
       #
-      class Instance < Entrypoint
+      class Instance < Template
+        include Bootable
+        extend Taggable
+
         tags :core
+
+        def perform(address, *files)
+          Runtime::Scenario.define(:gitlab_address, address)
+
+          ##
+          # Perform before hooks, which are different for CE and EE
+          #
+          Runtime::Release.perform_before_hooks
+
+          Specs::Runner.perform do |specs|
+            specs.tty = true
+            specs.tags = self.class.focus
+            specs.files = files.any? ? files : 'qa/specs/features'
+          end
+        end
       end
     end
   end
diff --git a/qa/qa/scenario/test/integration/mattermost.rb b/qa/qa/scenario/test/integration/mattermost.rb
index 7d0702afdb1..d939f52ab16 100644
--- a/qa/qa/scenario/test/integration/mattermost.rb
+++ b/qa/qa/scenario/test/integration/mattermost.rb
@@ -6,7 +6,7 @@ module QA
         # Run test suite against any GitLab instance where mattermost is enabled,
         # including staging and on-premises installation.
         #
-        class Mattermost < Scenario::Entrypoint
+        class Mattermost < Test::Instance
           tags :core, :mattermost
 
           def perform(address, mattermost, *files)
diff --git a/qa/qa/service/omnibus.rb b/qa/qa/service/omnibus.rb
new file mode 100644
index 00000000000..b5c06874e5c
--- /dev/null
+++ b/qa/qa/service/omnibus.rb
@@ -0,0 +1,20 @@
+module QA
+  module Service
+    class Omnibus
+      include Scenario::Actable
+      include Service::Shellout
+
+      def initialize(container)
+        @name = container
+      end
+
+      def gitlab_ctl(command, input: nil)
+        if input.nil?
+          shell "docker exec #{@name} gitlab-ctl #{command}"
+        else
+          shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'"
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb
new file mode 100644
index 00000000000..d0ee33c69f2
--- /dev/null
+++ b/qa/qa/service/runner.rb
@@ -0,0 +1,41 @@
+require 'securerandom'
+
+module QA
+  module Service
+    class Runner
+      include Scenario::Actable
+      include Service::Shellout
+
+      attr_accessor :token, :address, :tags, :image
+
+      def initialize(name)
+        @image = 'gitlab/gitlab-runner:alpine'
+        @name = name || "qa-runner-#{SecureRandom.hex(4)}"
+        @network = Runtime::Scenario.attributes[:network] || 'test'
+        @tags = %w[qa test]
+      end
+
+      def pull
+        shell "docker pull #{@image}"
+      end
+
+      def register!
+        shell <<~CMD.tr("\n", ' ')
+          docker run -d --rm --entrypoint=/bin/sh
+          --network #{@network} --name #{@name}
+          -e CI_SERVER_URL=#{@address}
+          -e REGISTER_NON_INTERACTIVE=true
+          -e REGISTRATION_TOKEN=#{@token}
+          -e RUNNER_EXECUTOR=shell
+          -e RUNNER_TAG_LIST=#{@tags.join(',')}
+          -e RUNNER_NAME=#{@name}
+          #{@image} -c 'gitlab-runner register && gitlab-runner run'
+        CMD
+      end
+
+      def remove!
+        shell "docker rm -f #{@name}"
+      end
+    end
+  end
+end
diff --git a/qa/qa/service/shellout.rb b/qa/qa/service/shellout.rb
new file mode 100644
index 00000000000..898febde63c
--- /dev/null
+++ b/qa/qa/service/shellout.rb
@@ -0,0 +1,23 @@
+require 'open3'
+
+module QA
+  module Service
+    module Shellout
+      ##
+      # TODO, make it possible to use generic QA framework classes
+      # as a library - gitlab-org/gitlab-qa#94
+      #
+      def shell(command)
+        puts "Executing `#{command}`"
+
+        Open3.popen2e(command) do |_in, out, wait|
+          out.each { |line| puts line }
+
+          if wait.value.exited? && wait.value.exitstatus.nonzero?
+            raise "Command `#{command}` failed!"
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/qa/qa/shell/omnibus.rb b/qa/qa/shell/omnibus.rb
deleted file mode 100644
index 6b3628d3109..00000000000
--- a/qa/qa/shell/omnibus.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-require 'open3'
-
-module QA
-  module Shell
-    class Omnibus
-      include Scenario::Actable
-
-      def initialize(container)
-        @name = container
-      end
-
-      def gitlab_ctl(command, input: nil)
-        if input.nil?
-          shell "docker exec #{@name} gitlab-ctl #{command}"
-        else
-          shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'"
-        end
-      end
-
-      private
-
-      ##
-      # TODO, make it possible to use generic QA framework classes
-      # as a library - gitlab-org/gitlab-qa#94
-      #
-      def shell(command)
-        puts "Executing `#{command}`"
-
-        Open3.popen2e(command) do |_in, out, wait|
-          out.each { |line| puts line }
-
-          if wait.value.exited? && wait.value.exitstatus.nonzero?
-            raise "Docker command `#{command}` failed!"
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb
new file mode 100644
index 00000000000..1bb7730e06c
--- /dev/null
+++ b/qa/qa/specs/features/project/pipelines_spec.rb
@@ -0,0 +1,102 @@
+module QA
+  feature 'CI/CD Pipelines', :core, :docker do
+    let(:executor) { "qa-runner-#{Time.now.to_i}" }
+
+    after do
+      Service::Runner.new(executor).remove!
+    end
+
+    scenario 'user registers a new specific runner' do
+      Runtime::Browser.visit(:gitlab, Page::Main::Login)
+      Page::Main::Login.act { sign_in_using_credentials }
+
+      Factory::Resource::Runner.fabricate! do |runner|
+        runner.name = executor
+      end
+
+      Page::Project::Settings::CICD.perform do |settings|
+        sleep 5 # Runner should register within 5 seconds
+        settings.refresh
+
+        settings.expand_runners_settings do |page|
+          expect(page).to have_content(executor)
+          expect(page).to have_online_runner
+        end
+      end
+    end
+
+    scenario 'users creates a new pipeline' do
+      Runtime::Browser.visit(:gitlab, Page::Main::Login)
+      Page::Main::Login.act { sign_in_using_credentials }
+
+      project = Factory::Resource::Project.fabricate! do |project|
+        project.name = 'project-with-pipelines'
+        project.description = 'Project with CI/CD Pipelines.'
+      end
+
+      Factory::Resource::Runner.fabricate! do |runner|
+        runner.project = project
+        runner.name = executor
+        runner.tags = %w[qa test]
+      end
+
+      Factory::Repository::Push.fabricate! do |push|
+        push.project = project
+        push.file_name = '.gitlab-ci.yml'
+        push.commit_message = 'Add .gitlab-ci.yml'
+        push.file_content = <<~EOF
+          test-success:
+            tags:
+              - qa
+              - test
+            script: echo 'OK'
+
+          test-failure:
+            tags:
+              - qa
+              - test
+            script:
+              - echo 'FAILURE'
+              - exit 1
+
+          test-tags:
+            tags:
+              - qa
+              - docker
+            script: echo 'NOOP'
+
+          test-artifacts:
+            tags:
+              - qa
+              - test
+            script: echo "CONTENTS" > my-artifacts/artifact.txt
+            artifacts:
+              paths:
+                - my-artifacts/
+        EOF
+      end
+
+      Page::Project::Show.act { wait_for_push }
+
+      expect(page).to have_content('Add .gitlab-ci.yml')
+
+      Page::Menu::Side.act { click_ci_cd_pipelines }
+
+      expect(page).to have_content('All 1')
+      expect(page).to have_content('Add .gitlab-ci.yml')
+
+      puts 'Waiting for the runner to process the pipeline'
+      sleep 15 # Runner should process all jobs within 15 seconds.
+
+      Page::Project::Pipeline::Index.act { go_to_latest_pipeline }
+
+      Page::Project::Pipeline::Show.perform do |pipeline|
+        expect(pipeline).to be_running
+        expect(pipeline).to have_build('test-success', status: :success)
+        expect(pipeline).to have_build('test-failure', status: :failed)
+        expect(pipeline).to have_build('test-tags', status: :pending)
+        expect(pipeline).to have_build('test-artifacts', status: :failed)
+      end
+    end
+  end
+end
diff --git a/qa/qa/specs/features/repository/push_spec.rb b/qa/qa/specs/features/repository/push_spec.rb
index 4f6ffe14c9f..51d9c2c7fd2 100644
--- a/qa/qa/specs/features/repository/push_spec.rb
+++ b/qa/qa/specs/features/repository/push_spec.rb
@@ -11,10 +11,7 @@ module QA
           push.commit_message = 'Add README.md'
         end
 
-        Page::Project::Show.act do
-          wait_for_push
-          refresh
-        end
+        Page::Project::Show.act { wait_for_push }
 
         expect(page).to have_content('README.md')
         expect(page).to have_content('This is a test project')
diff --git a/qa/spec/factory/base_spec.rb b/qa/spec/factory/base_spec.rb
index 90dd58e20fd..c5663049be8 100644
--- a/qa/spec/factory/base_spec.rb
+++ b/qa/spec/factory/base_spec.rb
@@ -19,7 +19,6 @@ describe QA::Factory::Base do
 
     it 'returns fabrication product' do
       allow(subject).to receive(:new).and_return(factory)
-      allow(factory).to receive(:fabricate!).and_return('something')
 
       result = subject.fabricate!('something')
 
diff --git a/qa/spec/scenario/entrypoint_spec.rb b/qa/spec/scenario/test/instance_spec.rb
index aec79dcea04..1824db54c9b 100644
--- a/qa/spec/scenario/entrypoint_spec.rb
+++ b/qa/spec/scenario/test/instance_spec.rb
@@ -1,6 +1,6 @@
-describe QA::Scenario::Entrypoint do
+describe QA::Scenario::Test::Instance do
   subject do
-    Class.new(QA::Scenario::Entrypoint) do
+    Class.new(described_class) do
       tags :rspec
     end
   end