Commit dcb79741 authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'qa/gb/ci-cd-qa-pipelines' into 'master'

CI/CD end-to-end tests

Closes gitlab-qa#18

See merge request gitlab-org/gitlab-ce!16619
parents 944c1eb6 5bead3e2
......@@ -8,7 +8,7 @@
= (_("(checkout the %{link} for information on how to install it).") % { link: link }).html_safe
%li
= _("Specify the following URL during the Runner setup:")
%code= root_url(only_path: false)
%code#coordinator_address= root_url(only_path: false)
%li
= _("Use the following registration token during setup:")
%code#registration_token= registration_token
......
......@@ -28,6 +28,7 @@ module Resource
autoload :Group, 'qa/factory/resource/group'
autoload :Project, 'qa/factory/resource/project'
autoload :DeployKey, 'qa/factory/resource/deploy_key'
autoload :Runner, 'qa/factory/resource/runner'
autoload :PersonalAccessToken, 'qa/factory/resource/personal_access_token'
end
......@@ -49,7 +50,7 @@ module Scenario
#
autoload :Bootable, 'qa/scenario/bootable'
autoload :Actable, 'qa/scenario/actable'
autoload :Entrypoint, 'qa/scenario/entrypoint'
autoload :Taggable, 'qa/scenario/taggable'
autoload :Template, 'qa/scenario/template'
##
......@@ -108,7 +109,14 @@ module Project
module Settings
autoload :Common, 'qa/page/project/settings/common'
autoload :Repository, 'qa/page/project/settings/repository'
autoload :CICD, 'qa/page/project/settings/ci_cd'
autoload :DeployKeys, 'qa/page/project/settings/deploy_keys'
autoload :Runners, 'qa/page/project/settings/runners'
end
module Pipeline
autoload :Index, 'qa/page/project/pipeline/index'
autoload :Show, 'qa/page/project/pipeline/show'
end
end
......@@ -134,10 +142,13 @@ module Git
end
##
# Classes describing shell interaction with GitLab
# Classes describing services being part of GitLab and how we can interact
# with these services, like through the shell.
#
module Shell
autoload :Omnibus, 'qa/shell/omnibus'
module Service
autoload :Shellout, 'qa/service/shellout'
autoload :Omnibus, 'qa/service/omnibus'
autoload :Runner, 'qa/service/runner'
end
##
......
......@@ -19,7 +19,7 @@ def fabricate!
project.visit!
Page::Menu::Side.act do
click_repository_setting
click_repository_settings
end
Page::Project::Settings::Repository.perform do |setting|
......
require 'securerandom'
module QA
module Factory
module Resource
class Runner < Factory::Base
attr_writer :name, :tags
dependency Factory::Resource::Project, as: :project do |project|
project.name = 'project-with-ci-cd'
project.description = 'Project with CI/CD Pipelines'
end
def name
@name || "qa-runner-#{SecureRandom.hex(4)}"
end
def tags
@tags || %w[qa e2e]
end
def fabricate!
project.visit!
Page::Menu::Side.act { click_ci_cd_settings }
Service::Runner.new(name).tap do |runner|
Page::Project::Settings::CICD.perform do |settings|
settings.expand_runners_settings do |runners|
runner.pull
runner.token = runners.registration_token
runner.address = runners.coordinator_address
runner.tags = tags
runner.register!
end
end
end
end
end
end
end
end
......@@ -5,18 +5,35 @@ class Side < Page::Base
view 'app/views/layouts/nav/sidebar/_project.html.haml' do
element :settings_item
element :repository_link, "title: 'Repository'"
element :pipelines_settings_link, "title: 'CI / CD'"
element :top_level_items, '.sidebar-top-level-items'
end
def click_repository_setting
hover_setting do
click_link('Repository')
def click_repository_settings
hover_settings do
within_submenu do
click_link('Repository')
end
end
end
def click_ci_cd_settings
hover_settings do
within_submenu do
click_link('CI / CD')
end
end
end
def click_ci_cd_pipelines
within_sidebar do
click_link('CI / CD')
end
end
private
def hover_setting
def hover_settings
within_sidebar do
find('.qa-settings-item').hover
......@@ -29,6 +46,12 @@ def within_sidebar
yield
end
end
def within_submenu
page.within('.fly-out-list') do
yield
end
end
end
end
end
......
module QA::Page
module Project::Pipeline
class Index < QA::Page::Base
view 'app/assets/javascripts/pipelines/components/pipeline_url.vue' do
element :pipeline_link, 'class="js-pipeline-url-link"'
end
def go_to_latest_pipeline
first('.js-pipeline-url-link').click
end
end
end
end
module QA::Page
module Project::Pipeline
class Show < QA::Page::Base
view 'app/assets/javascripts/vue_shared/components/header_ci_component.vue' do
element :pipeline_header, /header class.*ci-header-container.*/
end
view 'app/assets/javascripts/pipelines/components/graph/graph_component.vue' do
element :pipeline_graph, /class.*pipeline-graph.*/
end
view 'app/assets/javascripts/pipelines/components/graph/job_component.vue' do
element :job_component, /class.*ci-job-component.*/
end
view 'app/assets/javascripts/vue_shared/components/ci_icon.vue' do
element :status_icon, 'ci-status-icon-${status}'
end
def running?
within('.ci-header-container') do
return page.has_content?('running')
end
end
def has_build?(name, status: :success)
within('.pipeline-graph') do
within('.ci-job-component', text: name) do
return has_selector?(".ci-status-icon-#{status}")
end
end
end
end
end
end
module QA
module Page
module Project
module Settings
class CICD < Page::Base
include Common
view 'app/views/projects/settings/ci_cd/show.html.haml' do
element :runners_settings, 'Runners settings'
end
def expand_runners_settings(&block)
expand_section('Runners settings') do
Settings::Runners.perform(&block)
end
end
end
end
end
end
end
......@@ -10,6 +10,16 @@ def expand(element_name)
yield
end
end
def expand_section(name)
page.within('#content-body') do
page.within('section', text: name) do
click_button 'Expand'
yield
end
end
end
end
end
end
......
module QA
module Page
module Project
module Settings
class Runners < Page::Base
view 'app/views/ci/runner/_how_to_setup_runner.html.haml' do
element :registration_token, '%code#registration_token'
element :coordinator_address, '%code#coordinator_address'
end
##
# TODO, phase-out CSS classes added in Ruby helpers.
#
view 'app/helpers/runners_helper.rb' do
# rubocop:disable Lint/InterpolationCheck
element :runner_status, 'runner-status-#{status}'
# rubocop:enable Lint/InterpolationCheck
end
def registration_token
find('code#registration_token').text
end
def coordinator_address
find('code#coordinator_address').text
end
def has_online_runner?
page.has_css?('.runner-status-online')
end
end
end
end
end
end
......@@ -33,6 +33,7 @@ def project_name
def wait_for_push
sleep 5
refresh
end
end
end
......
module QA
module Scenario
##
# Base class for running the suite against any GitLab instance,
# including staging and on-premises installation.
#
class Entrypoint < Template
include Bootable
def perform(address, *files)
Runtime::Scenario.define(:gitlab_address, address)
##
# Perform before hooks, which are different for CE and EE
#
Runtime::Release.perform_before_hooks
Specs::Runner.perform do |specs|
specs.tty = true
specs.tags = self.class.get_tags
specs.files = files.any? ? files : 'qa/specs/features'
end
end
def self.tags(*tags)
@tags = tags
end
def self.get_tags
@tags
end
end
end
end
module QA
module Scenario
module Taggable
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def tags(*tags)
@tags = tags
end
def focus
@tags.to_a
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
......@@ -2,11 +2,29 @@ module QA
module Scenario
module Test
##
# Run test suite against any GitLab instance,
# Base class for running the suite against any GitLab instance,
# including staging and on-premises installation.
#
class Instance < Entrypoint
class Instance < Template
include Bootable
extend Taggable
tags :core
def perform(address, *files)
Runtime::Scenario.define(:gitlab_address, address)
##
# Perform before hooks, which are different for CE and EE
#
Runtime::Release.perform_before_hooks
Specs::Runner.perform do |specs|
specs.tty = true
specs.tags = self.class.focus
specs.files = files.any? ? files : 'qa/specs/features'
end
end
end
end
end
......
......@@ -6,7 +6,7 @@ module Integration
# Run test suite against any GitLab instance where mattermost is enabled,
# including staging and on-premises installation.
#
class Mattermost < Scenario::Entrypoint
class Mattermost < Test::Instance
tags :core, :mattermost
def perform(address, mattermost, *files)
......
module QA
module Service
class Omnibus
include Scenario::Actable
include Service::Shellout
def initialize(container)
@name = container
end
def gitlab_ctl(command, input: nil)
if input.nil?
shell "docker exec #{@name} gitlab-ctl #{command}"
else
shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'"
end
end
end
end
end
require 'securerandom'
module QA
module Service
class Runner
include Scenario::Actable
include Service::Shellout
attr_accessor :token, :address, :tags, :image
def initialize(name)
@image = 'gitlab/gitlab-runner:alpine'
@name = name || "qa-runner-#{SecureRandom.hex(4)}"
@network = Runtime::Scenario.attributes[:network] || 'test'
@tags = %w[qa test]
end
def pull
shell "docker pull #{@image}"
end
def register!
shell <<~CMD.tr("\n", ' ')
docker run -d --rm --entrypoint=/bin/sh
--network #{@network} --name #{@name}
-e CI_SERVER_URL=#{@address}
-e REGISTER_NON_INTERACTIVE=true
-e REGISTRATION_TOKEN=#{@token}
-e RUNNER_EXECUTOR=shell
-e RUNNER_TAG_LIST=#{@tags.join(',')}
-e RUNNER_NAME=#{@name}
#{@image} -c 'gitlab-runner register && gitlab-runner run'
CMD
end
def remove!
shell "docker rm -f #{@name}"
end
end
end
end
require 'open3'
module QA
module Shell
class Omnibus
include Scenario::Actable
def initialize(container)
@name = container
end
def gitlab_ctl(command, input: nil)
if input.nil?
shell "docker exec #{@name} gitlab-ctl #{command}"
else
shell "docker exec #{@name} bash -c '#{input} | gitlab-ctl #{command}'"
end
end
private
module Service
module Shellout
##
# TODO, make it possible to use generic QA framework classes
# as a library - gitlab-org/gitlab-qa#94
......@@ -30,7 +14,7 @@ def shell(command)
out.each { |line| puts line }
if wait.value.exited? && wait.value.exitstatus.nonzero?
raise "Docker command `#{command}` failed!"
raise "Command `#{command}` failed!"
end
end
end
......
module QA
feature 'CI/CD Pipelines', :core, :docker do
let(:executor) { "qa-runner-#{Time.now.to_i}" }
after do
Service::Runner.new(executor).remove!
end
scenario 'user registers a new specific runner' do
Runtime::Browser.visit(:gitlab, Page::Main::Login)
Page::Main::Login.act { sign_in_using_credentials }
Factory::Resource::Runner.fabricate! do |runner|
runner.name = executor
end
Page::Project::Settings::CICD.perform do |settings|
sleep 5 # Runner should register within 5 seconds
settings.refresh
settings.expand_runners_settings do |page|
expect(page).to have_content(executor)
expect(page).to have_online_runner
end
end
end
scenario 'users creates a new pipeline' do
Runtime::Browser.visit(:gitlab, Page::Main::Login)
Page::Main::Login.act { sign_in_using_credentials }
project = Factory::Resource::Project.fabricate! do |project|
project.name = 'project-with-pipelines'
project.description = 'Project with CI/CD Pipelines.'
end
Factory::Resource::Runner.fabricate! do |runner|
runner.project = project
runner.name = executor
runner.tags = %w[qa test]
end
Factory::Repository::Push.fabricate! do |push|
push.project = project
push.file_name = '.gitlab-ci.yml'
push.commit_message = 'Add .gitlab-ci.yml'
push.file_content = <<~EOF
test-success:
tags:
- qa
- test
script: echo 'OK'
test-failure:
tags:
- qa
- test
script:
- echo 'FAILURE'
- exit 1
test-tags:
tags:
- qa
- docker
script: echo 'NOOP'
test-artifacts:
tags:
- qa
- test
script: echo "CONTENTS" > my-artifacts/artifact.txt
artifacts:
paths:
- my-artifacts/
EOF
end
Page::Project::Show.act { wait_for_push }
expect(page).to have_content('Add .gitlab-ci.yml')
Page::Menu::Side.act { click_ci_cd_pipelines }
expect(page).to have_content('All 1')
expect(page).to have_content('Add .gitlab-ci.yml')
puts 'Waiting for the runner to process the pipeline'
sleep 15 # Runner should process all jobs within 15 seconds.
Page::Project::Pipeline::Index.act { go_to_latest_pipeline }
Page::Project::Pipeline::Show.perform do |pipeline|
expect(pipeline).to be_running
expect(pipeline).to have_build('test-success', status: :success)
expect(pipeline).to have_build('test-failure', status: :failed)
expect(pipeline).to have_build('test-tags', status: :pending)
expect(pipeline).to have_build('test-artifacts', status: :failed)
end
end
end
end
......@@ -11,10 +11,7 @@ module QA
push.commit_message = 'Add README.md'
end
Page::Project::Show.act do
wait_for_push
refresh
end
Page::Project::Show.act { wait_for_push }
expect(page).to have_content('README.md')
expect(page).to have_content('This is a test project')
......
......@@ -19,7 +19,6 @@
it 'returns fabrication product' do
allow(subject).to receive(:new).and_return(factory)
allow(factory).to receive(:fabricate!).and_return('something')
result = subject.fabricate!('something')
......