GitLab steht aufgrund wichtiger Wartungsarbeiten am Montag, den 8. März, zwischen 17:00 und 19:00 Uhr nicht zur Verfügung.

Commit 4766a77b authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'feature/gb/auto-retry-failed-ci-job' into 'master'

Make it possible to auto retry a failed CI/CD job

Closes #3442

See merge request !12909
parents 001dd56e 2f620aa7
......@@ -96,6 +96,14 @@ def retry(build, current_user)
BuildSuccessWorker.perform_async(id)
end
end
before_transition any => [:failed] do |build|
next if build.retries_max.zero?
if build.retries_count < build.retries_max
Ci::Build.retry(build, build.user)
end
end
end
def detailed_status(current_user)
......@@ -130,6 +138,14 @@ def retryable?
success? || failed? || canceled?
end
def retries_count
pipeline.builds.retried.where(name: self.name).count
end
def retries_max
self.options.fetch(:retry, 0).to_i
end
def latest?
!retried?
end
......
---
title: Allow to configure automatic retry of a failed CI/CD job
merge_request: 12909
author:
......@@ -395,6 +395,7 @@ job_name:
| after_script | no | Override a set of commands that are executed after job |
| environment | no | Defines a name of environment to which deployment is done by this job |
| coverage | no | Define code coverage settings for a given job |
| retry | no | Define how many times a job can be auto-retried in case of a failure |
### script
......@@ -1129,9 +1130,33 @@ A simple example:
```yaml
job1:
script: rspec
coverage: '/Code coverage: \d+\.\d+/'
```
### retry
**Notes:**
- [Introduced][ce-3442] in GitLab 9.5.
`retry` allows you to configure how many times a job is going to be retried in
case of a failure.
When a job fails, and has `retry` configured it is going to be processed again
up to the amount of times specified by the `retry` keyword.
If `retry` is set to 2, and a job succeeds in a second run (first retry), it won't be retried
again. `retry` value has to be a positive integer, equal or larger than 0, but
lower or equal to 2 (two retries maximum, three runs in total).
A simple example:
```yaml
test:
script: rspec
retry: 2
```
## Git Strategy
> Introduced in GitLab 8.9 as an experimental feature. May change or be removed
......@@ -1506,3 +1531,4 @@ CI with various languages.
[variables]: ../variables/README.md
[ce-7983]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7983
[ce-7447]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7447
[ce-3442]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3442
......@@ -83,7 +83,8 @@ def build_attributes(name)
before_script: job[:before_script],
script: job[:script],
after_script: job[:after_script],
environment: job[:environment]
environment: job[:environment],
retry: job[:retry]
}.compact }
end
......
......@@ -11,7 +11,7 @@ class Job < Node
ALLOWED_KEYS = %i[tags script only except type image services allow_failure
type stage when artifacts cache dependencies before_script
after_script variables environment coverage].freeze
after_script variables environment coverage retry].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS
......@@ -23,6 +23,9 @@ class Job < Node
with_options allow_nil: true do
validates :tags, array_of_strings: true
validates :allow_failure, boolean: true
validates :retry, numericality: { only_integer: true,
greater_than_or_equal_to: 0,
less_than_or_equal_to: 2 }
validates :when,
inclusion: { in: %w[on_success on_failure always manual],
message: 'should be on_success, on_failure, ' \
......@@ -76,9 +79,9 @@ class Job < Node
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage
:artifacts, :commands, :environment, :coverage, :retry
attributes :script, :tags, :allow_failure, :when, :dependencies
attributes :script, :tags, :allow_failure, :when, :dependencies, :retry
def compose!(deps = nil)
super do
......@@ -142,6 +145,7 @@ def to_hash
environment: environment_defined? ? environment_value : nil,
environment_name: environment_defined? ? environment_value[:name] : nil,
coverage: coverage_defined? ? coverage_value : nil,
retry: retry_defined? ? retry_value.to_i : nil,
artifacts: artifacts_value,
after_script: after_script_value,
ignore: ignored? }
......
......@@ -84,6 +84,10 @@
success
end
trait :retried do
retried true
end
trait :cancelable do
pending
end
......
......@@ -32,6 +32,28 @@ module Ci
end
end
describe 'retry entry' do
context 'when retry count is specified' do
let(:config) do
YAML.dump(rspec: { script: 'rspec', retry: 1 })
end
it 'includes retry count in build options attribute' do
expect(subject[:options]).to include(retry: 1)
end
end
context 'when retry count is not specified' do
let(:config) do
YAML.dump(rspec: { script: 'rspec' })
end
it 'does not persist retry count in the database' do
expect(subject[:options]).not_to have_key(:retry)
end
end
end
describe 'allow failure entry' do
context 'when job is a manual action' do
context 'when allow_failure is defined' do
......
......@@ -80,6 +80,45 @@
expect(entry.errors).to include "job script can't be blank"
end
end
context 'when retry value is not correct' do
context 'when it is not a numeric value' do
let(:config) { { retry: true } }
it 'returns error about invalid type' do
expect(entry).not_to be_valid
expect(entry.errors).to include 'job retry is not a number'
end
end
context 'when it is lower than zero' do
let(:config) { { retry: -1 } }
it 'returns error about value too low' do
expect(entry).not_to be_valid
expect(entry.errors)
.to include 'job retry must be greater than or equal to 0'
end
end
context 'when it is not an integer' do
let(:config) { { retry: 1.5 } }
it 'returns error about wrong value' do
expect(entry).not_to be_valid
expect(entry.errors).to include 'job retry must be an integer'
end
end
context 'when the value is too high' do
let(:config) { { retry: 10 } }
it 'returns error about value too high' do
expect(entry).not_to be_valid
expect(entry.errors).to include 'job retry must be less than or equal to 2'
end
end
end
end
end
......
......@@ -802,6 +802,47 @@
end
end
describe 'build auto retry feature' do
describe '#retries_count' do
subject { create(:ci_build, name: 'test', pipeline: pipeline) }
context 'when build has been retried several times' do
before do
create(:ci_build, :retried, name: 'test', pipeline: pipeline)
create(:ci_build, :retried, name: 'test', pipeline: pipeline)
end
it 'reports a correct retry count value' do
expect(subject.retries_count).to eq 2
end
end
context 'when build has not been retried' do
it 'returns zero' do
expect(subject.retries_count).to eq 0
end
end
end
describe '#retries_max' do
context 'when max retries value is defined' do
subject { create(:ci_build, options: { retry: 1 }) }
it 'returns a number of configured max retries' do
expect(subject.retries_max).to eq 1
end
end
context 'when max retries value is not defined' do
subject { create(:ci_build) }
it 'returns zero' do
expect(subject.retries_max).to eq 0
end
end
end
end
describe '#keep_artifacts!' do
let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) }
......@@ -1583,7 +1624,7 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now)
end
end
describe 'State transition: any => [:pending]' do
describe 'state transition: any => [:pending]' do
let(:build) { create(:ci_build, :created) }
it 'queues BuildQueueWorker' do
......@@ -1592,4 +1633,35 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now)
build.enqueue
end
end
describe 'state transition when build fails' do
context 'when build is configured to be retried' do
subject { create(:ci_build, :running, options: { retry: 3 }) }
it 'retries builds and assigns a same user to it' do
expect(described_class).to receive(:retry)
.with(subject, subject.user)
subject.drop!
end
end
context 'when build is not configured to be retried' do
subject { create(:ci_build, :running) }
it 'does not retry build' do
expect(described_class).not_to receive(:retry)
subject.drop!
end
it 'does not count retries when not necessary' do
expect(described_class).not_to receive(:retry)
expect_any_instance_of(described_class)
.not_to receive(:retries_count)
subject.drop!
end
end
end
end
......@@ -320,5 +320,19 @@ def previous_commit_sha_from_ref(ref)
end.not_to change { Environment.count }
end
end
context 'when builds with auto-retries are configured' do
before do
config = YAML.dump(rspec: { script: 'rspec', retry: 2 })
stub_ci_pipeline_yaml_file(config)
end
it 'correctly creates builds with auto-retry value configured' do
pipeline = execute_service
expect(pipeline).to be_persisted
expect(pipeline.builds.find_by(name: 'rspec').retries_max).to eq 2
end
end
end
end
......@@ -463,6 +463,35 @@
end
end
context 'when builds with auto-retries are configured' do
before do
create_build('build:1', stage_idx: 0, user: user, options: { retry: 2 })
create_build('test:1', stage_idx: 1, user: user, when: :on_failure)
create_build('test:2', stage_idx: 1, user: user, options: { retry: 1 })
end
it 'automatically retries builds in a valid order' do
expect(process_pipeline).to be_truthy
fail_running_or_pending
expect(builds_names).to eq %w[build:1 build:1]
expect(builds_statuses).to eq %w[failed pending]
succeed_running_or_pending
expect(builds_names).to eq %w[build:1 build:1 test:2]
expect(builds_statuses).to eq %w[failed success pending]
succeed_running_or_pending
expect(builds_names).to eq %w[build:1 build:1 test:2]
expect(builds_statuses).to eq %w[failed success success]
expect(pipeline.reload).to be_success
end
end
def process_pipeline
described_class.new(pipeline.project, user).execute(pipeline)
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment