From d0b08f1c50b0b4106c0ae5f398912aab596a447a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 22 Jan 2018 09:59:55 +0100 Subject: Add GitLab Runner service to GitLab QA --- qa/qa.rb | 9 ++++++--- qa/qa/service/omnibus.rb | 20 ++++++++++++++++++++ qa/qa/service/runner.rb | 28 ++++++++++++++++++++++++++++ qa/qa/service/shellout.rb | 23 +++++++++++++++++++++++ qa/qa/shell/omnibus.rb | 39 --------------------------------------- 5 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 qa/qa/service/omnibus.rb create mode 100644 qa/qa/service/runner.rb create mode 100644 qa/qa/service/shellout.rb delete mode 100644 qa/qa/shell/omnibus.rb diff --git a/qa/qa.rb b/qa/qa.rb index 4803432aeee..5c277b95517 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -126,10 +126,13 @@ module QA end ## - # Classes describing shell interaction with GitLab + # Classes describing services being part of GitLab and how we can interact + # with these services, like through the shell. # - module Shell - autoload :Omnibus, 'qa/shell/omnibus' + module Service + autoload :Shellable, 'qa/service/shellable' + autoload :Omnibus, 'qa/service/omnibus' + autoload :Runner, 'qa/service/runner' end ## diff --git a/qa/qa/service/omnibus.rb b/qa/qa/service/omnibus.rb new file mode 100644 index 00000000000..b5c06874e5c --- /dev/null +++ b/qa/qa/service/omnibus.rb @@ -0,0 +1,20 @@ +module QA + module Service + class Omnibus + include Scenario::Actable + include Service::Shellout + + def initialize(container) + @name = container + end + + def gitlab_ctl(command, input: nil) + if input.nil? + shell "docker exec #{@name} gitlab-ctl #{command}" + else + shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'" + end + end + end + end +end diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb new file mode 100644 index 00000000000..a6a60f53787 --- /dev/null +++ b/qa/qa/service/runner.rb @@ -0,0 +1,28 @@ +module QA + module Service + class Runner + include Scenario::Actable + include Service::Shellout + + def initialize(image) + @image = image + end + + def pull + shell "docker pull #{@image}" + end + + def register(token) + raise NotImplementedError + end + + def run + raise NotImplementedError + end + + def remove + raise NotImplementedError + end + end + end +end diff --git a/qa/qa/service/shellout.rb b/qa/qa/service/shellout.rb new file mode 100644 index 00000000000..898febde63c --- /dev/null +++ b/qa/qa/service/shellout.rb @@ -0,0 +1,23 @@ +require 'open3' + +module QA + module Service + module Shellout + ## + # TODO, make it possible to use generic QA framework classes + # as a library - gitlab-org/gitlab-qa#94 + # + def shell(command) + puts "Executing `#{command}`" + + Open3.popen2e(command) do |_in, out, wait| + out.each { |line| puts line } + + if wait.value.exited? && wait.value.exitstatus.nonzero? + raise "Command `#{command}` failed!" + end + end + end + end + end +end diff --git a/qa/qa/shell/omnibus.rb b/qa/qa/shell/omnibus.rb deleted file mode 100644 index 6b3628d3109..00000000000 --- a/qa/qa/shell/omnibus.rb +++ /dev/null @@ -1,39 +0,0 @@ -require 'open3' - -module QA - module Shell - class Omnibus - include Scenario::Actable - - def initialize(container) - @name = container - end - - def gitlab_ctl(command, input: nil) - if input.nil? - shell "docker exec #{@name} gitlab-ctl #{command}" - else - shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'" - end - end - - private - - ## - # TODO, make it possible to use generic QA framework classes - # as a library - gitlab-org/gitlab-qa#94 - # - def shell(command) - puts "Executing `#{command}`" - - Open3.popen2e(command) do |_in, out, wait| - out.each { |line| puts line } - - if wait.value.exited? && wait.value.exitstatus.nonzero? - raise "Docker command `#{command}` failed!" - end - end - end - end - end -end -- cgit v1.2.1 From 50a82f64e53d396ca91f52c4d25af60beb52a424 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 22 Jan 2018 14:03:03 +0100 Subject: Add end-to-end test for registering GitLab Runner --- qa/qa.rb | 5 ++- qa/qa/factory/resource/deploy_key.rb | 2 +- qa/qa/factory/resource/runner.rb | 52 ++++++++++++++++++++++++++ qa/qa/page/menu/side.rb | 25 +++++++++++-- qa/qa/page/project/settings/ci_cd.rb | 21 +++++++++++ qa/qa/page/project/settings/common.rb | 10 +++++ qa/qa/page/project/settings/runners.rb | 19 ++++++++++ qa/qa/service/runner.rb | 31 ++++++++++----- qa/qa/specs/features/project/pipelines_spec.rb | 17 +++++++++ 9 files changed, 166 insertions(+), 16 deletions(-) create mode 100644 qa/qa/factory/resource/runner.rb create mode 100644 qa/qa/page/project/settings/ci_cd.rb create mode 100644 qa/qa/page/project/settings/runners.rb create mode 100644 qa/qa/specs/features/project/pipelines_spec.rb diff --git a/qa/qa.rb b/qa/qa.rb index 5c277b95517..24738b5531a 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -26,6 +26,7 @@ module QA autoload :Group, 'qa/factory/resource/group' autoload :Project, 'qa/factory/resource/project' autoload :DeployKey, 'qa/factory/resource/deploy_key' + autoload :Runner, 'qa/factory/resource/runner' end module Repository @@ -104,7 +105,9 @@ module QA module Settings autoload :Common, 'qa/page/project/settings/common' autoload :Repository, 'qa/page/project/settings/repository' + autoload :CICD, 'qa/page/project/settings/ci_cd' autoload :DeployKeys, 'qa/page/project/settings/deploy_keys' + autoload :Runners, 'qa/page/project/settings/runners' end end @@ -130,7 +133,7 @@ module QA # with these services, like through the shell. # module Service - autoload :Shellable, 'qa/service/shellable' + autoload :Shellout, 'qa/service/shellout' autoload :Omnibus, 'qa/service/omnibus' autoload :Runner, 'qa/service/runner' end diff --git a/qa/qa/factory/resource/deploy_key.rb b/qa/qa/factory/resource/deploy_key.rb index 7c58e70bcc4..d51701e65ea 100644 --- a/qa/qa/factory/resource/deploy_key.rb +++ b/qa/qa/factory/resource/deploy_key.rb @@ -13,7 +13,7 @@ module QA project.visit! Page::Menu::Side.act do - click_repository_setting + click_repository_settings end Page::Project::Settings::Repository.perform do |setting| diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb new file mode 100644 index 00000000000..037136dd113 --- /dev/null +++ b/qa/qa/factory/resource/runner.rb @@ -0,0 +1,52 @@ +require 'securerandom' + +module QA + module Factory + module Resource + class Runner < Factory::Base + attr_writer :name + + dependency Factory::Resource::Project, as: :project do |project| + project.name = 'project-with-ci-cd' + project.description = 'Project with CI/CD Pipelines' + end + + def name + @name || "qa-runner-#{SecureRandom.hex(4)}" + end + + def perform(&block) + @block ||= block + end + + def fabricate! + project.visit! + + Page::Menu::Side.act { click_ci_cd_settings } + + Service::Runner.perform do |runner| + Page::Project::Settings::CICD.perform do |settings| + settings.expand_runners_settings do |runners| + runner.pull + runner.name = name + runner.token = runners.registration_token + runner.address = runners.coordinator_address + runner.tags = %w[qa test] + runner.register! + end + + sleep 5 # TODO, non-blocking waiting for Runner to register. + + settings.refresh + + settings.expand_runners_settings do |runners| + perform&.call(runners) + runner.remove! + end + end + end + end + end + end + end +end diff --git a/qa/qa/page/menu/side.rb b/qa/qa/page/menu/side.rb index 1df4e0c2429..e666d570172 100644 --- a/qa/qa/page/menu/side.rb +++ b/qa/qa/page/menu/side.rb @@ -5,18 +5,29 @@ module QA view 'app/views/layouts/nav/sidebar/_project.html.haml' do element :settings_item element :repository_link, "title: 'Repository'" + element :repository_link, "title: 'CI / CD'" element :top_level_items, '.sidebar-top-level-items' end - def click_repository_setting - hover_setting do - click_link('Repository') + def click_repository_settings + hover_settings do + within_submenu do + click_link('Repository') + end + end + end + + def click_ci_cd_settings + hover_settings do + within_submenu do + click_link('CI / CD') + end end end private - def hover_setting + def hover_settings within_sidebar do find('.qa-settings-item').hover @@ -29,6 +40,12 @@ module QA yield end end + + def within_submenu + page.within('.fly-out-list') do + yield + end + end end end end diff --git a/qa/qa/page/project/settings/ci_cd.rb b/qa/qa/page/project/settings/ci_cd.rb new file mode 100644 index 00000000000..5270dde7411 --- /dev/null +++ b/qa/qa/page/project/settings/ci_cd.rb @@ -0,0 +1,21 @@ +module QA + module Page + module Project + module Settings + class CICD < Page::Base + include Common + + view 'app/views/projects/settings/ci_cd/show.html.haml' do + element :runners_settings, 'Runners settings' + end + + def expand_runners_settings(&block) + expand_section('Runners settings') do + Settings::Runners.perform(&block) + end + end + end + end + end + end +end diff --git a/qa/qa/page/project/settings/common.rb b/qa/qa/page/project/settings/common.rb index b4ef07e1540..1357bf031d5 100644 --- a/qa/qa/page/project/settings/common.rb +++ b/qa/qa/page/project/settings/common.rb @@ -10,6 +10,16 @@ module QA yield end end + + def expand_section(name) + page.within('#content-body') do + page.within('section', text: name) do + click_button 'Expand' + + yield + end + end + end end end end diff --git a/qa/qa/page/project/settings/runners.rb b/qa/qa/page/project/settings/runners.rb new file mode 100644 index 00000000000..ba3dc632c59 --- /dev/null +++ b/qa/qa/page/project/settings/runners.rb @@ -0,0 +1,19 @@ +module QA + module Page + module Project + module Settings + class Runners < Page::Base + def registration_token + find('code#registration_token').text + end + + def coordinator_address + # TODO, this needs a specific ID or QA class + # + all('code').first.text + end + end + end + end + end +end diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb index a6a60f53787..a9906b8627a 100644 --- a/qa/qa/service/runner.rb +++ b/qa/qa/service/runner.rb @@ -1,27 +1,38 @@ +require 'securerandom' + module QA module Service class Runner include Scenario::Actable include Service::Shellout - def initialize(image) - @image = image + attr_writer :token, :address, :tags, :image, :name + + def initialize + @image = 'gitlab/gitlab-runner:alpine' + @name = "gitlab-runner-qa-#{SecureRandom.hex(4)}" end def pull shell "docker pull #{@image}" end - def register(token) - raise NotImplementedError - end - - def run - raise NotImplementedError + def register! + shell <<~CMD.tr("\n", ' ') + docker run -d --rm --entrypoint=/bin/sh + --network test --name #{@name} + -e CI_SERVER_URL=#{@address} + -e REGISTER_NON_INTERACTIVE=true + -e REGISTRATION_TOKEN=#{@token} + -e RUNNER_EXECUTOR=shell + -e RUNNER_TAG_LIST=#{@tags.to_a.join(',')} + -e RUNNER_NAME=#{@name} + #{@image} -c 'gitlab-runner register && gitlab-runner run' + CMD end - def remove - raise NotImplementedError + def remove! + shell "docker rm -f #{@name}" end end end diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb new file mode 100644 index 00000000000..1897b2fa27c --- /dev/null +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -0,0 +1,17 @@ +module QA + feature 'CI/CD Pipelines', :core, :docker do + scenario 'user registers a new specific runner' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + Factory::Resource::Runner.fabricate! do |runner| + runner.name = 'my-qa-runner' + + runner.perform do |page| + expect(page).to have_content('my-qa-runner') + expect(page).to have_css('.runner-status-online') + end + end + end + end +end -- cgit v1.2.1 From ca4db9d25f797a07bed3750ca9d8f0833487f55c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 22 Jan 2018 14:12:36 +0100 Subject: Wait for Runner to register in QA and DRY page objects --- qa/qa/factory/resource/runner.rb | 9 ++++++--- qa/qa/page/project/settings/runners.rb | 4 ++++ qa/qa/service/runner.rb | 8 ++++++-- qa/qa/specs/features/project/pipelines_spec.rb | 8 +++----- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb index 037136dd113..9d11f3c572f 100644 --- a/qa/qa/factory/resource/runner.rb +++ b/qa/qa/factory/resource/runner.rb @@ -35,12 +35,15 @@ module QA runner.register! end - sleep 5 # TODO, non-blocking waiting for Runner to register. - + ## + # TODO, refactor to support non-blocking wait time until + # GitLab Runner sucessfully registers itself. + # + sleep 5 settings.refresh settings.expand_runners_settings do |runners| - perform&.call(runners) + perform&.call(runners, runner) runner.remove! end end diff --git a/qa/qa/page/project/settings/runners.rb b/qa/qa/page/project/settings/runners.rb index ba3dc632c59..ac93c3efddd 100644 --- a/qa/qa/page/project/settings/runners.rb +++ b/qa/qa/page/project/settings/runners.rb @@ -12,6 +12,10 @@ module QA # all('code').first.text end + + def has_online_runner? + page.has_css?('.runner-status-online') + end end end end diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb index a9906b8627a..2745bf00ded 100644 --- a/qa/qa/service/runner.rb +++ b/qa/qa/service/runner.rb @@ -6,11 +6,11 @@ module QA include Scenario::Actable include Service::Shellout - attr_writer :token, :address, :tags, :image, :name + attr_accessor :token, :address, :tags, :image, :name def initialize @image = 'gitlab/gitlab-runner:alpine' - @name = "gitlab-runner-qa-#{SecureRandom.hex(4)}" + @name = "qa-runner-#{SecureRandom.hex(4)}" end def pull @@ -18,6 +18,10 @@ module QA end def register! + ## + # TODO, this assumes that `test` network exists, because we know that + # gitlab-qa environment orchestration tool creates it. + # shell <<~CMD.tr("\n", ' ') docker run -d --rm --entrypoint=/bin/sh --network test --name #{@name} diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index 1897b2fa27c..84e5a0e93cf 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -5,11 +5,9 @@ module QA Page::Main::Login.act { sign_in_using_credentials } Factory::Resource::Runner.fabricate! do |runner| - runner.name = 'my-qa-runner' - - runner.perform do |page| - expect(page).to have_content('my-qa-runner') - expect(page).to have_css('.runner-status-online') + runner.perform do |page, runner| + expect(page).to have_content(runner.name) + expect(page).to have_online_runner end end end -- cgit v1.2.1 From 5520e3ebc329cdfec4bf61aa29dd25de563c3e43 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 22 Jan 2018 15:18:19 +0100 Subject: Reduce nesting and simplify CI/CD end-to-end tests --- qa/qa/factory/resource/runner.rb | 21 ++--------- qa/qa/service/runner.rb | 6 +-- qa/qa/specs/features/project/pipelines_spec.rb | 52 +++++++++++++++++++++++++- qa/spec/factory/base_spec.rb | 1 - 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb index 9d11f3c572f..299a9d0716d 100644 --- a/qa/qa/factory/resource/runner.rb +++ b/qa/qa/factory/resource/runner.rb @@ -15,36 +15,21 @@ module QA @name || "qa-runner-#{SecureRandom.hex(4)}" end - def perform(&block) - @block ||= block - end - def fabricate! project.visit! Page::Menu::Side.act { click_ci_cd_settings } - Service::Runner.perform do |runner| + Service::Runner.new(name).tap do |runner| Page::Project::Settings::CICD.perform do |settings| settings.expand_runners_settings do |runners| runner.pull - runner.name = name runner.token = runners.registration_token runner.address = runners.coordinator_address runner.tags = %w[qa test] runner.register! - end - - ## - # TODO, refactor to support non-blocking wait time until - # GitLab Runner sucessfully registers itself. - # - sleep 5 - settings.refresh - - settings.expand_runners_settings do |runners| - perform&.call(runners, runner) - runner.remove! + # TODO, wait for runner to register using non-blocking method. + sleep 5 end end end diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb index 2745bf00ded..2e313b59d28 100644 --- a/qa/qa/service/runner.rb +++ b/qa/qa/service/runner.rb @@ -6,11 +6,11 @@ module QA include Scenario::Actable include Service::Shellout - attr_accessor :token, :address, :tags, :image, :name + attr_accessor :token, :address, :tags, :image - def initialize + def initialize(name) @image = 'gitlab/gitlab-runner:alpine' - @name = "qa-runner-#{SecureRandom.hex(4)}" + @name = name || "qa-runner-#{SecureRandom.hex(4)}" end def pull diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index 84e5a0e93cf..56a6acee6a4 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -1,15 +1,63 @@ module QA feature 'CI/CD Pipelines', :core, :docker do + let(:executor) { "qa-runner-#{Time.now.to_i}" } + + after do + Service::Runner.new(executor).remove! + end + scenario 'user registers a new specific runner' do Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.act { sign_in_using_credentials } Factory::Resource::Runner.fabricate! do |runner| - runner.perform do |page, runner| - expect(page).to have_content(runner.name) + runner.name = executor + end + + Page::Project::Settings::CICD.perform do |settings| + settings.expand_runners_settings do |page| + expect(page).to have_content(runner) expect(page).to have_online_runner end end end + + scenario 'users creates a new pipeline' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + project = Factory::Resource::Project.fabricate! do |project| + project.name = 'project-with-pipelines' + project.description = 'Project with CI/CD Pipelines.' + end + + Factory::Resource::Runner.fabricate! do |runner| + runner.project = project + runner.name = executor + end + + Factory::Repository::Push.fabricate! do |push| + push.project = project + push.file_name = '.gitlab-ci.yml' + push.commit_message = 'Add .gitlab-ci.yml' + push.file_content = <<~EOF + echo-success-test: + script: echo 'OK' + + echo-failure-test: + script: + - echo 'FAILURE' + - exit 1 + + echo-artifacts-test: + script: echo "CONTENTS" > my-artifacts/artifact.txt + artifacts: + paths: + - my-artifacts/ + EOF + end + + Page::Project::Show.act { wait_for_push } + end end end diff --git a/qa/spec/factory/base_spec.rb b/qa/spec/factory/base_spec.rb index 90dd58e20fd..c5663049be8 100644 --- a/qa/spec/factory/base_spec.rb +++ b/qa/spec/factory/base_spec.rb @@ -19,7 +19,6 @@ describe QA::Factory::Base do it 'returns fabrication product' do allow(subject).to receive(:new).and_return(factory) - allow(factory).to receive(:fabricate!).and_return('something') result = subject.fabricate!('something') -- cgit v1.2.1 From 738bad8ee83932ffd1b7b4de9113b0152b37172a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 22 Jan 2018 15:31:41 +0100 Subject: Fix CI/CD end-to-end tests by waiting for changes --- qa/qa/page/project/show.rb | 1 + qa/qa/specs/features/project/pipelines_spec.rb | 6 +++++- qa/qa/specs/features/repository/push_spec.rb | 5 +---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb index c8af5ba6280..5e66e40a0b5 100644 --- a/qa/qa/page/project/show.rb +++ b/qa/qa/page/project/show.rb @@ -33,6 +33,7 @@ module QA def wait_for_push sleep 5 + refresh end end end diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index 56a6acee6a4..f9280cd049a 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -15,8 +15,10 @@ module QA end Page::Project::Settings::CICD.perform do |settings| + settings.refresh + settings.expand_runners_settings do |page| - expect(page).to have_content(runner) + expect(page).to have_content(executor) expect(page).to have_online_runner end end @@ -58,6 +60,8 @@ module QA end Page::Project::Show.act { wait_for_push } + + expect(page).to have_content('Add .gitlab-ci.yml') end end end diff --git a/qa/qa/specs/features/repository/push_spec.rb b/qa/qa/specs/features/repository/push_spec.rb index 4f6ffe14c9f..51d9c2c7fd2 100644 --- a/qa/qa/specs/features/repository/push_spec.rb +++ b/qa/qa/specs/features/repository/push_spec.rb @@ -11,10 +11,7 @@ module QA push.commit_message = 'Add README.md' end - Page::Project::Show.act do - wait_for_push - refresh - end + Page::Project::Show.act { wait_for_push } expect(page).to have_content('README.md') expect(page).to have_content('This is a test project') -- cgit v1.2.1 From 5cb94835c0010d1954b26c858f910d533cd88e38 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 12:04:01 +0100 Subject: Refactor QA instance test scenario and tags --- qa/qa.rb | 2 +- qa/qa/scenario/entrypoint.rb | 34 --------------------- qa/qa/scenario/taggable.rb | 13 ++++++++ qa/qa/scenario/test/instance.rb | 22 ++++++++++++-- qa/qa/scenario/test/integration/mattermost.rb | 2 +- qa/qa/service/runner.rb | 10 +++--- qa/spec/scenario/entrypoint_spec.rb | 44 --------------------------- qa/spec/scenario/test/instance_spec.rb | 44 +++++++++++++++++++++++++++ 8 files changed, 83 insertions(+), 88 deletions(-) delete mode 100644 qa/qa/scenario/entrypoint.rb create mode 100644 qa/qa/scenario/taggable.rb delete mode 100644 qa/spec/scenario/entrypoint_spec.rb create mode 100644 qa/spec/scenario/test/instance_spec.rb diff --git a/qa/qa.rb b/qa/qa.rb index 24738b5531a..2766fc988a3 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -47,7 +47,7 @@ module QA # autoload :Bootable, 'qa/scenario/bootable' autoload :Actable, 'qa/scenario/actable' - autoload :Entrypoint, 'qa/scenario/entrypoint' + autoload :Taggable, 'qa/scenario/taggable' autoload :Template, 'qa/scenario/template' ## diff --git a/qa/qa/scenario/entrypoint.rb b/qa/qa/scenario/entrypoint.rb deleted file mode 100644 index ae099fd911e..00000000000 --- a/qa/qa/scenario/entrypoint.rb +++ /dev/null @@ -1,34 +0,0 @@ -module QA - module Scenario - ## - # Base class for running the suite against any GitLab instance, - # including staging and on-premises installation. - # - class Entrypoint < Template - include Bootable - - def perform(address, *files) - Runtime::Scenario.define(:gitlab_address, address) - - ## - # Perform before hooks, which are different for CE and EE - # - Runtime::Release.perform_before_hooks - - Specs::Runner.perform do |specs| - specs.tty = true - specs.tags = self.class.get_tags - specs.files = files.any? ? files : 'qa/specs/features' - end - end - - def self.tags(*tags) - @tags = tags - end - - def self.get_tags - @tags - end - end - end -end diff --git a/qa/qa/scenario/taggable.rb b/qa/qa/scenario/taggable.rb new file mode 100644 index 00000000000..b63c245bf47 --- /dev/null +++ b/qa/qa/scenario/taggable.rb @@ -0,0 +1,13 @@ +module QA + module Scenario + module Taggable + def tags(*tags) + @tags = tags + end + + def focus + @tags.to_a + end + end + end +end diff --git a/qa/qa/scenario/test/instance.rb b/qa/qa/scenario/test/instance.rb index e2a1f6bf2bd..993bbd723a3 100644 --- a/qa/qa/scenario/test/instance.rb +++ b/qa/qa/scenario/test/instance.rb @@ -2,11 +2,29 @@ module QA module Scenario module Test ## - # Run test suite against any GitLab instance, + # Base class for running the suite against any GitLab instance, # including staging and on-premises installation. # - class Instance < Entrypoint + class Instance < Template + include Bootable + extend Taggable + tags :core + + def perform(address, *files) + Runtime::Scenario.define(:gitlab_address, address) + + ## + # Perform before hooks, which are different for CE and EE + # + Runtime::Release.perform_before_hooks + + Specs::Runner.perform do |specs| + specs.tty = true + specs.tags = self.class.focus + specs.files = files.any? ? files : 'qa/specs/features' + end + end end end end diff --git a/qa/qa/scenario/test/integration/mattermost.rb b/qa/qa/scenario/test/integration/mattermost.rb index 7d0702afdb1..d939f52ab16 100644 --- a/qa/qa/scenario/test/integration/mattermost.rb +++ b/qa/qa/scenario/test/integration/mattermost.rb @@ -6,7 +6,7 @@ module QA # Run test suite against any GitLab instance where mattermost is enabled, # including staging and on-premises installation. # - class Mattermost < Scenario::Entrypoint + class Mattermost < Test::Instance tags :core, :mattermost def perform(address, mattermost, *files) diff --git a/qa/qa/service/runner.rb b/qa/qa/service/runner.rb index 2e313b59d28..d0ee33c69f2 100644 --- a/qa/qa/service/runner.rb +++ b/qa/qa/service/runner.rb @@ -11,6 +11,8 @@ module QA def initialize(name) @image = 'gitlab/gitlab-runner:alpine' @name = name || "qa-runner-#{SecureRandom.hex(4)}" + @network = Runtime::Scenario.attributes[:network] || 'test' + @tags = %w[qa test] end def pull @@ -18,18 +20,14 @@ module QA end def register! - ## - # TODO, this assumes that `test` network exists, because we know that - # gitlab-qa environment orchestration tool creates it. - # shell <<~CMD.tr("\n", ' ') docker run -d --rm --entrypoint=/bin/sh - --network test --name #{@name} + --network #{@network} --name #{@name} -e CI_SERVER_URL=#{@address} -e REGISTER_NON_INTERACTIVE=true -e REGISTRATION_TOKEN=#{@token} -e RUNNER_EXECUTOR=shell - -e RUNNER_TAG_LIST=#{@tags.to_a.join(',')} + -e RUNNER_TAG_LIST=#{@tags.join(',')} -e RUNNER_NAME=#{@name} #{@image} -c 'gitlab-runner register && gitlab-runner run' CMD diff --git a/qa/spec/scenario/entrypoint_spec.rb b/qa/spec/scenario/entrypoint_spec.rb deleted file mode 100644 index aec79dcea04..00000000000 --- a/qa/spec/scenario/entrypoint_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -describe QA::Scenario::Entrypoint do - subject do - Class.new(QA::Scenario::Entrypoint) do - tags :rspec - end - end - - context '#perform' do - let(:arguments) { spy('Runtime::Scenario') } - let(:release) { spy('Runtime::Release') } - let(:runner) { spy('Specs::Runner') } - - before do - stub_const('QA::Runtime::Release', release) - stub_const('QA::Runtime::Scenario', arguments) - stub_const('QA::Specs::Runner', runner) - - allow(runner).to receive(:perform).and_yield(runner) - end - - it 'sets an address of the subject' do - subject.perform("hello") - - expect(arguments).to have_received(:define) - .with(:gitlab_address, "hello") - end - - context 'no paths' do - it 'should call runner with default arguments' do - subject.perform("test") - - expect(runner).to have_received(:files=).with('qa/specs/features') - end - end - - context 'specifying paths' do - it 'should call runner with paths' do - subject.perform('test', 'path1', 'path2') - - expect(runner).to have_received(:files=).with(%w[path1 path2]) - end - end - end -end diff --git a/qa/spec/scenario/test/instance_spec.rb b/qa/spec/scenario/test/instance_spec.rb new file mode 100644 index 00000000000..1824db54c9b --- /dev/null +++ b/qa/spec/scenario/test/instance_spec.rb @@ -0,0 +1,44 @@ +describe QA::Scenario::Test::Instance do + subject do + Class.new(described_class) do + tags :rspec + end + end + + context '#perform' do + let(:arguments) { spy('Runtime::Scenario') } + let(:release) { spy('Runtime::Release') } + let(:runner) { spy('Specs::Runner') } + + before do + stub_const('QA::Runtime::Release', release) + stub_const('QA::Runtime::Scenario', arguments) + stub_const('QA::Specs::Runner', runner) + + allow(runner).to receive(:perform).and_yield(runner) + end + + it 'sets an address of the subject' do + subject.perform("hello") + + expect(arguments).to have_received(:define) + .with(:gitlab_address, "hello") + end + + context 'no paths' do + it 'should call runner with default arguments' do + subject.perform("test") + + expect(runner).to have_received(:files=).with('qa/specs/features') + end + end + + context 'specifying paths' do + it 'should call runner with paths' do + subject.perform('test', 'path1', 'path2') + + expect(runner).to have_received(:files=).with(%w[path1 path2]) + end + end + end +end -- cgit v1.2.1 From 294bddf95c646e08f68aeec16024b48e18dd3ba2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 12:10:18 +0100 Subject: Use runner tags when processing CI/CD QA pipelines --- qa/qa/factory/resource/runner.rb | 8 ++++++-- qa/qa/specs/features/project/pipelines_spec.rb | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb index 299a9d0716d..109e2e59fcc 100644 --- a/qa/qa/factory/resource/runner.rb +++ b/qa/qa/factory/resource/runner.rb @@ -4,7 +4,7 @@ module QA module Factory module Resource class Runner < Factory::Base - attr_writer :name + attr_writer :name, :tags dependency Factory::Resource::Project, as: :project do |project| project.name = 'project-with-ci-cd' @@ -15,6 +15,10 @@ module QA @name || "qa-runner-#{SecureRandom.hex(4)}" end + def tags + @tags || %w[qa e2e] + end + def fabricate! project.visit! @@ -26,7 +30,7 @@ module QA runner.pull runner.token = runners.registration_token runner.address = runners.coordinator_address - runner.tags = %w[qa test] + runner.tags = tags runner.register! # TODO, wait for runner to register using non-blocking method. sleep 5 diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index f9280cd049a..3a6c93cd903 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -36,6 +36,7 @@ module QA Factory::Resource::Runner.fabricate! do |runner| runner.project = project runner.name = executor + runner.tags = %w[qa test] end Factory::Repository::Push.fabricate! do |push| @@ -44,14 +45,23 @@ module QA push.commit_message = 'Add .gitlab-ci.yml' push.file_content = <<~EOF echo-success-test: + tags: + - qa + - test script: echo 'OK' echo-failure-test: + tags: + - qa + - test script: - echo 'FAILURE' - exit 1 echo-artifacts-test: + tags: + - qa + - test script: echo "CONTENTS" > my-artifacts/artifact.txt artifacts: paths: -- cgit v1.2.1 From 2fe5d9327965cb18fe800a0c7bd3f83c44a70efb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 12:14:07 +0100 Subject: Remove blocking wait-time for runner to register in QA --- qa/qa/factory/resource/runner.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/qa/factory/resource/runner.rb b/qa/qa/factory/resource/runner.rb index 109e2e59fcc..5f37f8ac2e9 100644 --- a/qa/qa/factory/resource/runner.rb +++ b/qa/qa/factory/resource/runner.rb @@ -32,8 +32,6 @@ module QA runner.address = runners.coordinator_address runner.tags = tags runner.register! - # TODO, wait for runner to register using non-blocking method. - sleep 5 end end end -- cgit v1.2.1 From ededa488f2c94d1d132d974476f38e668c44e4ee Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 12:55:44 +0100 Subject: Assert on pipeline jobs statuses in CI/CD QA tests --- qa/qa.rb | 5 +++++ qa/qa/page/menu/side.rb | 8 ++++++- qa/qa/page/project/pipeline/index.rb | 13 +++++++++++ qa/qa/page/project/pipeline/show.rb | 23 ++++++++++++++++++++ qa/qa/specs/features/project/pipelines_spec.rb | 30 +++++++++++++++++++++++--- 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 qa/qa/page/project/pipeline/index.rb create mode 100644 qa/qa/page/project/pipeline/show.rb diff --git a/qa/qa.rb b/qa/qa.rb index 2766fc988a3..b1d13ec6ab9 100644 --- a/qa/qa.rb +++ b/qa/qa.rb @@ -109,6 +109,11 @@ module QA autoload :DeployKeys, 'qa/page/project/settings/deploy_keys' autoload :Runners, 'qa/page/project/settings/runners' end + + module Pipeline + autoload :Index, 'qa/page/project/pipeline/index' + autoload :Show, 'qa/page/project/pipeline/show' + end end module Admin diff --git a/qa/qa/page/menu/side.rb b/qa/qa/page/menu/side.rb index e666d570172..7f0f924c5e8 100644 --- a/qa/qa/page/menu/side.rb +++ b/qa/qa/page/menu/side.rb @@ -5,7 +5,7 @@ module QA view 'app/views/layouts/nav/sidebar/_project.html.haml' do element :settings_item element :repository_link, "title: 'Repository'" - element :repository_link, "title: 'CI / CD'" + element :pipelines_settings_link, "title: 'CI / CD'" element :top_level_items, '.sidebar-top-level-items' end @@ -25,6 +25,12 @@ module QA end end + def click_ci_cd_pipelines + within_sidebar do + click_link('CI / CD') + end + end + private def hover_settings diff --git a/qa/qa/page/project/pipeline/index.rb b/qa/qa/page/project/pipeline/index.rb new file mode 100644 index 00000000000..32c108393b9 --- /dev/null +++ b/qa/qa/page/project/pipeline/index.rb @@ -0,0 +1,13 @@ +module QA::Page + module Project::Pipeline + class Index < QA::Page::Base + view 'app/assets/javascripts/pipelines/components/pipeline_url.vue' do + element :pipeline_link, 'class="js-pipeline-url-link"' + end + + def go_to_latest_pipeline + first('.js-pipeline-url-link').click + end + end + end +end diff --git a/qa/qa/page/project/pipeline/show.rb b/qa/qa/page/project/pipeline/show.rb new file mode 100644 index 00000000000..309d9e75ea2 --- /dev/null +++ b/qa/qa/page/project/pipeline/show.rb @@ -0,0 +1,23 @@ +module QA::Page + module Project::Pipeline + class Show < QA::Page::Base + view 'app/assets/javascripts/vue_shared/components/header_ci_component.vue' do + element :pipeline_header, /header class.*ci-header-container.*/ + end + + def running? + within('.ci-header-container') do + return page.has_content?('running') + end + end + + def has_build?(name, status: :success) + within('.pipeline-graph') do + within('.ci-job-component', text: name) do + return has_selector?(".ci-status-icon-#{status}") + end + end + end + end + end +end diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index 3a6c93cd903..6760605df72 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -44,13 +44,13 @@ module QA push.file_name = '.gitlab-ci.yml' push.commit_message = 'Add .gitlab-ci.yml' push.file_content = <<~EOF - echo-success-test: + test-success: tags: - qa - test script: echo 'OK' - echo-failure-test: + test-failure: tags: - qa - test @@ -58,7 +58,13 @@ module QA - echo 'FAILURE' - exit 1 - echo-artifacts-test: + test-tags: + tags: + - qa + - docker + script: echo 'NOOP' + + test-artifacts: tags: - qa - test @@ -72,6 +78,24 @@ module QA Page::Project::Show.act { wait_for_push } expect(page).to have_content('Add .gitlab-ci.yml') + + Page::Menu::Side.act { click_ci_cd_pipelines } + + expect(page).to have_content('All 1') + expect(page).to have_content('Add .gitlab-ci.yml') + + puts 'Waiting for the runner to process the pipeline' + sleep 15 # Runner should process all jobs within 15 seconds. + + Page::Project::Pipeline::Index.act { go_to_latest_pipeline } + + Page::Project::Pipeline::Show.perform do |pipeline| + expect(pipeline).to be_running + expect(pipeline).to have_build('test-success', status: :success) + expect(pipeline).to have_build('test-failure', status: :failed) + expect(pipeline).to have_build('test-tags', status: :pending) + expect(pipeline).to have_build('test-artifacts', status: :failed) + end end end end -- cgit v1.2.1 From d327277c39df1537b41073912d9b4e9f765d8da5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 13:02:35 +0100 Subject: Add views / selectors for pipeline show page object --- qa/qa/page/project/pipeline/show.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qa/qa/page/project/pipeline/show.rb b/qa/qa/page/project/pipeline/show.rb index 309d9e75ea2..0835173f1cd 100644 --- a/qa/qa/page/project/pipeline/show.rb +++ b/qa/qa/page/project/pipeline/show.rb @@ -5,6 +5,18 @@ module QA::Page element :pipeline_header, /header class.*ci-header-container.*/ end + view 'app/assets/javascripts/pipelines/components/graph/graph_component.vue' do + element :pipeline_graph, /class.*pipeline-graph.*/ + end + + view 'app/assets/javascripts/pipelines/components/graph/job_component.vue' do + element :job_component, /class.*ci-job-component.*/ + end + + view 'app/assets/javascripts/vue_shared/components/ci_icon.vue' do + element :status_icon, 'ci-status-icon-${status}' + end + def running? within('.ci-header-container') do return page.has_content?('running') -- cgit v1.2.1 From 915b554773244a883790308fd9608f5f301edde0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 13:09:12 +0100 Subject: Add specific views / selectors for QA runners page --- app/views/ci/runner/_how_to_setup_runner.html.haml | 2 +- qa/qa/page/project/settings/runners.rb | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/views/ci/runner/_how_to_setup_runner.html.haml b/app/views/ci/runner/_how_to_setup_runner.html.haml index b75dab0acc5..8db7727b80c 100644 --- a/app/views/ci/runner/_how_to_setup_runner.html.haml +++ b/app/views/ci/runner/_how_to_setup_runner.html.haml @@ -8,7 +8,7 @@ = (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe %li = _("Specify the following URL during the Runner setup:") - %code= root_url(only_path: false) + %code#coordinator_address= root_url(only_path: false) %li = _("Use the following registration token during setup:") %code#registration_token= registration_token diff --git a/qa/qa/page/project/settings/runners.rb b/qa/qa/page/project/settings/runners.rb index ac93c3efddd..ff763402d98 100644 --- a/qa/qa/page/project/settings/runners.rb +++ b/qa/qa/page/project/settings/runners.rb @@ -3,14 +3,24 @@ module QA module Project module Settings class Runners < Page::Base + view 'app/views/ci/runner/_how_to_setup_runner.html.haml' do + element :registration_token, '%code#registration_token' + element :coordinator_address, '%code#coordinator_address' + end + + ## + # TODO, phase-out CSS classes from Ruby helpers. + # + view 'app/helpers/runners_helper.rb' do + element :runner_status, 'runner-status-#{status}' + end + def registration_token find('code#registration_token').text end def coordinator_address - # TODO, this needs a specific ID or QA class - # - all('code').first.text + find('code#coordinator_address').text end def has_online_runner? -- cgit v1.2.1 From 55ba4604390242de9728e47673f61703fbbaa02e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 13:53:33 +0100 Subject: Fix static-analysis offenses in QA support class --- qa/qa/scenario/taggable.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qa/qa/scenario/taggable.rb b/qa/qa/scenario/taggable.rb index b63c245bf47..b1f24d742e0 100644 --- a/qa/qa/scenario/taggable.rb +++ b/qa/qa/scenario/taggable.rb @@ -1,6 +1,8 @@ module QA module Scenario module Taggable + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def tags(*tags) @tags = tags end @@ -8,6 +10,8 @@ module QA def focus @tags.to_a end + + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end -- cgit v1.2.1 From a4e581229e729c5f21fa65cfc708e964b65b6c5b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 13:57:07 +0100 Subject: Wait for runner until it registers itself in QA tests --- qa/qa/specs/features/project/pipelines_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/qa/specs/features/project/pipelines_spec.rb b/qa/qa/specs/features/project/pipelines_spec.rb index 6760605df72..1bb7730e06c 100644 --- a/qa/qa/specs/features/project/pipelines_spec.rb +++ b/qa/qa/specs/features/project/pipelines_spec.rb @@ -15,6 +15,7 @@ module QA end Page::Project::Settings::CICD.perform do |settings| + sleep 5 # Runner should register within 5 seconds settings.refresh settings.expand_runners_settings do |page| -- cgit v1.2.1 From 5bead3e29965ec26ccf3320d65a5757d673d9958 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 23 Jan 2018 14:31:32 +0100 Subject: Fix offense in runners settings QA page object class --- qa/qa/page/project/settings/runners.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qa/qa/page/project/settings/runners.rb b/qa/qa/page/project/settings/runners.rb index ff763402d98..b41668c94cd 100644 --- a/qa/qa/page/project/settings/runners.rb +++ b/qa/qa/page/project/settings/runners.rb @@ -9,10 +9,12 @@ module QA end ## - # TODO, phase-out CSS classes from Ruby helpers. + # TODO, phase-out CSS classes added in Ruby helpers. # view 'app/helpers/runners_helper.rb' do + # rubocop:disable Lint/InterpolationCheck element :runner_status, 'runner-status-#{status}' + # rubocop:enable Lint/InterpolationCheck end def registration_token -- cgit v1.2.1