Commit 73547da0 authored by Sanad Liaquat's avatar Sanad Liaquat Committed by GitLab Release Tools Bot
Browse files

Merge branch 'qa-ml-quarantine-context' into 'master'

[QA] Skip contexts in quarantine

See merge request gitlab-org/gitlab-ce!24942

(cherry picked from commit c6016ac8)

a2229074 Skip contexts in quarantine
a3f93880 Refactor rspec config quarantine before blocks
parent a35f16c4
......@@ -17,26 +17,16 @@
end
end
config.before do |example|
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
# If quarantine is tagged, skip tests that have other metadata unless
# they're also tagged. This lets us run quarantined tests in a particular
# category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
if config.inclusion_filter[:quarantine]
skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config)
config.before(:context) do
if self.class.metadata.keys.include?(:quarantine)
skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
end
end
config.before(:each, :quarantine) do |example|
# Skip tests in quarantine unless we explicitly focus on them
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
skip('In quarantine') unless config.inclusion_filter[:quarantine]
config.before do |example|
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
end
config.expect_with :rspec do |expectations|
......@@ -55,18 +45,41 @@
Kernel.srand config.seed
end
# Skip tests in quarantine unless we explicitly focus on them.
# Skip the entire context if a context is tagged. This avoids running before
# blocks unnecessarily.
# If quarantine is focussed, skip tests/contexts that have other metadata
# unless they're also focussed. This lets us run quarantined tests in a
# particular category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
included_filters = filters_other_than_quarantine(filter_keys)
if filter_keys.include?(:quarantine)
skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
else
skip('In quarantine') if metadata_keys.include?(:quarantine)
end
end
def filters_other_than_quarantine(filter_keys)
filter_keys.reject { |key| key == :quarantine }
end
# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
#
# Returns true if
# - the example metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(example, config)
return false unless example.metadata.keys.include? :quarantine
filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
return true if filters_other_than_quarantine.empty?
# - the metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
return false unless metadata_keys.include? :quarantine
return true if included_filters.empty?
filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key }
included_filters.any? { |key| metadata_keys.include? key }
end
......@@ -10,79 +10,79 @@
end
end
context 'default' do
it_behaves_like 'passing tests'
end
context 'foo', :foo do
it_behaves_like 'passing tests'
end
context 'default' do
context 'quarantine', :quarantine do
it_behaves_like 'passing tests'
end
context 'bar quarantine', :bar, :quarantine do
it_behaves_like 'passing tests'
end
end
end
context 'default config' do
it 'tests are skipped if in quarantine' do
context 'with no tags focussed' do
before do
group.run
end
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context 'in a context tagged :foo' do
it 'skips tests in quarantine' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
ex = default_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context 'in an untagged context' do
it 'skips tests in quarantine' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context "with 'quarantine' tagged" do
before do
RSpec.configure do |config|
config.inclusion_filter = :quarantine
end
end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
it "only quarantined tests are run" do
group.run
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to be(1)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to be(1)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
end
context "with 'foo' tagged" do
context 'with :quarantine focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = :foo
config.inclusion_filter = :quarantine
end
group.run
......@@ -93,30 +93,50 @@
end
end
it "tests are not run if not tagged 'foo'" do
default_context = group.children.find { |c| c.description == "default" }
expect(default_context.descendant_filtered_examples.count).to eq(0)
context 'in an untagged context' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
it "tests are skipped if in quarantine" do
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
context 'in a context tagged :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
end
context "with 'quarantine' and 'foo' tagged" do
context 'with a non-quarantine tag (:foo) focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = { quarantine: true, foo: true }
config.inclusion_filter = :foo
end
group.run
end
after do
RSpec.configure do |config|
......@@ -124,38 +144,43 @@
end
end
it 'of tests tagged foo, only tests in quarantine run' do
group.run
context 'in an untagged context' do
it 'runs no tests' do
context = group.children.find { |c| c.description == "default" }
expect(context.descendant_filtered_examples.count).to eq(0)
end
end
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
context 'in a context tagged :foo' do
it 'skips quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
it 'if tests are not tagged they are skipped, even if they are in quarantine' do
group.run
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(1)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
context 'in a context tagged :quarantine' do
it 'runs no tests' do
context = group.children.find { |c| c.description == "quarantine" }
expect(context.descendant_filtered_examples.count).to eq(0)
end
end
end
context "with 'foo' and 'bar' tagged" do
context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true }
config.inclusion_filter = { quarantine: true, foo: true }
end
group.run
end
after do
RSpec.configure do |config|
......@@ -163,67 +188,67 @@
end
end
it "runs tests tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar', :bar do
end
example 'foo and bar', :foo, :bar do
end
end
group.run
expect(group.examples.count).to eq(3)
context 'in an untagged context' do
it 'ignores untagged tests and skips tests even if in quarantine' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(1)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
ex = group.examples.find { |e| e.description == "bar" }
expect(ex.execution_result.status).to eq(:passed)
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:passed)
end
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
it "skips quarantined tests tagged 'foo' and/or 'bar'" do
group = RSpec.describe do
example 'foo in quarantine', :foo, :quarantine do
end
example 'foo and bar in quarantine', :foo, :bar, :quarantine do
end
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
group.run
expect(group.examples.count).to eq(2)
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = group.examples.find { |e| e.description == "foo and bar in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'in quarantine', :quarantine do
end
end
context 'in a context tagged :bar and :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
group.run
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = group.examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to be_nil
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end
end
end
context "with 'foo' and 'bar' and 'quarantined' tagged" do
context 'with :quarantine and multiple non-quarantine tags focussed' do
before do
RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true, quarantine: true }
end
group.run
end
after do
RSpec.configure do |config|
......@@ -231,34 +256,49 @@
end
end
it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do
group = RSpec.describe do
example 'foo', :foo do
end
example 'bar and quarantine', :bar, :quarantine do
end
example 'foo and bar', :foo, :bar do
end
example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
end
context 'in a context tagged :foo' do
it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end
end
group.run
expect(group.examples.count).to eq(4)
context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
ex = group.examples.find { |e| e.description == "bar and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end
end
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]')
context 'in a context tagged :bar and :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "bar quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment