Commit af4d4e70 authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Merge branch 'feature/sm/34834-missing-dependency-should-fail-job-2' into 'master'

Dependency validator

Closes #34834

See merge request gitlab-org/gitlab-ce!14009
parents 1c02af27 2ac6d806
...@@ -6,6 +6,8 @@ class Build < CommitStatus ...@@ -6,6 +6,8 @@ class Build < CommitStatus
include Presentable include Presentable
include Importable include Importable
MissingDependenciesError = Class.new(StandardError)
belongs_to :runner belongs_to :runner
belongs_to :trigger_request belongs_to :trigger_request
belongs_to :erased_by, class_name: 'User' belongs_to :erased_by, class_name: 'User'
...@@ -139,6 +141,10 @@ def retry(build, current_user) ...@@ -139,6 +141,10 @@ def retry(build, current_user)
Ci::Build.retry(build, build.user) Ci::Build.retry(build, build.user)
end end
end end
before_transition any => [:running] do |build|
build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies')
end
end end
def detailed_status(current_user) def detailed_status(current_user)
...@@ -478,6 +484,20 @@ def empty_dependencies? ...@@ -478,6 +484,20 @@ def empty_dependencies?
options[:dependencies]&.empty? options[:dependencies]&.empty?
end end
def validates_dependencies!
dependencies.each do |dependency|
raise MissingDependenciesError unless dependency.valid_dependency?
end
end
def valid_dependency?
return false unless complete?
return false if artifacts_expired?
return false if erased?
true
end
def hide_secrets(trace) def hide_secrets(trace)
return unless trace return unless trace
......
...@@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base ...@@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base
script_failure: 1, script_failure: 1,
api_failure: 2, api_failure: 2,
stuck_or_timeout_failure: 3, stuck_or_timeout_failure: 3,
runner_system_failure: 4 runner_system_failure: 4,
missing_dependency_failure: 5
} }
## ##
......
...@@ -54,6 +54,9 @@ def execute ...@@ -54,6 +54,9 @@ def execute
# we still have to return 409 in the end, # we still have to return 409 in the end,
# to make sure that this is properly handled by runner. # to make sure that this is properly handled by runner.
valid = false valid = false
rescue Ci::Build::MissingDependenciesError
build.drop!(:missing_dependency_failure)
valid = false
end end
end end
......
---
title: Fail jobs if its dependency is missing
merge_request: 14009
author:
type: fixed
...@@ -128,6 +128,45 @@ steps below. ...@@ -128,6 +128,45 @@ steps below.
1. Save the file and [restart GitLab][] for the changes to take effect. 1. Save the file and [restart GitLab][] for the changes to take effect.
## Validation for dependencies
> Introduced in GitLab 10.3.
To disable [the dependencies validation](../ci/yaml/README.md#when-a-dependent-job-will-fail),
you can flip the feature flag from a Rails console.
---
**In Omnibus installations:**
1. Enter the Rails console:
```sh
sudo gitlab-rails console
```
1. Flip the switch and disable it:
```ruby
Feature.enable('ci_disable_validates_dependencies')
```
---
**In installations from source:**
1. Enter the Rails console:
```sh
cd /home/git/gitlab
RAILS_ENV=production sudo -u git -H bundle exec rails console
```
1. Flip the switch and disable it:
```ruby
Feature.enable('ci_disable_validates_dependencies')
```
## Set the maximum file size of the artifacts ## Set the maximum file size of the artifacts
Provided the artifacts are enabled, you can change the maximum file size of the Provided the artifacts are enabled, you can change the maximum file size of the
......
...@@ -1153,6 +1153,20 @@ deploy: ...@@ -1153,6 +1153,20 @@ deploy:
script: make deploy script: make deploy
``` ```
#### When a dependent job will fail
> Introduced in GitLab 10.3.
If the artifacts of the job that is set as a dependency have been
[expired](#artifacts-expire_in) or
[erased](../../user/project/pipelines/job_artifacts.md#erasing-artifacts), then
the dependent job will fail.
NOTE: **Note:**
You can ask your administrator to
[flip this switch](../../administration/job_artifacts.md#validation-for-dependencies)
and bring back the old behavior.
### before_script and after_script ### before_script and after_script
It's possible to overwrite the globally defined `before_script` and `after_script`: It's possible to overwrite the globally defined `before_script` and `after_script`:
......
...@@ -44,7 +44,7 @@ the artifacts will be kept forever. ...@@ -44,7 +44,7 @@ the artifacts will be kept forever.
For more examples on artifacts, follow the [artifacts reference in For more examples on artifacts, follow the [artifacts reference in
`.gitlab-ci.yml`](../../../ci/yaml/README.md#artifacts). `.gitlab-ci.yml`](../../../ci/yaml/README.md#artifacts).
## Browsing job artifacts ## Browsing artifacts
>**Note:** >**Note:**
With GitLab 9.2, PDFs, images, videos and other formats can be previewed With GitLab 9.2, PDFs, images, videos and other formats can be previewed
...@@ -77,7 +77,7 @@ one HTML file that you can view directly online when ...@@ -77,7 +77,7 @@ one HTML file that you can view directly online when
--- ---
## Downloading job artifacts ## Downloading artifacts
If you need to download the whole archive, there are buttons in various places If you need to download the whole archive, there are buttons in various places
inside GitLab that make that possible. inside GitLab that make that possible.
...@@ -102,7 +102,7 @@ inside GitLab that make that possible. ...@@ -102,7 +102,7 @@ inside GitLab that make that possible.
![Job artifacts browser](img/job_artifacts_browser.png) ![Job artifacts browser](img/job_artifacts_browser.png)
## Downloading the latest job artifacts ## Downloading the latest artifacts
It is possible to download the latest artifacts of a job via a well known URL It is possible to download the latest artifacts of a job via a well known URL
so you can use it for scripting purposes. so you can use it for scripting purposes.
...@@ -163,6 +163,18 @@ information in the UI. ...@@ -163,6 +163,18 @@ information in the UI.
![Latest artifacts button](img/job_latest_artifacts_browser.png) ![Latest artifacts button](img/job_latest_artifacts_browser.png)
## Erasing artifacts
DANGER: **Warning:**
This is a destructive action that leads to data loss. Use with caution.
If you have at least Developer [permissions](../../permissions.md#gitlab-ci-cd-permissions)
on the project, you can erase a single job via the UI which will also remove the
artifacts and the job's trace.
1. Navigate to a job's page.
1. Click the trash icon at the top right of the job's trace.
1. Confirm the deletion.
[expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in [expiry date]: ../../../ci/yaml/README.md#artifacts-expire_in
[ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399 [ce-14399]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14399
...@@ -1868,6 +1868,94 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now) ...@@ -1868,6 +1868,94 @@ def create_mr(build, pipeline, factory: :merge_request, created_at: Time.now)
end end
end end
describe 'state transition: any => [:running]' do
shared_examples 'validation is active' do
context 'when depended job has not been completed yet' do
let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) }
end
context 'when artifacts of depended job has been expired' do
let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) }
end
context 'when artifacts of depended job has been erased' do
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }
before do
pre_stage_job.erase
end
it { expect { job.run! }.to raise_error(Ci::Build::MissingDependenciesError) }
end
end
shared_examples 'validation is not active' do
context 'when depended job has not been completed yet' do
let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect { job.run! }.not_to raise_error }
end
context 'when artifacts of depended job has been expired' do
let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect { job.run! }.not_to raise_error }
end
context 'when artifacts of depended job has been erased' do
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }
before do
pre_stage_job.erase
end
it { expect { job.run! }.not_to raise_error }
end
end
let!(:job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: options) }
context 'when validates for dependencies is enabled' do
before do
stub_feature_flags(ci_disable_validates_dependencies: false)
end
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) }
context 'when "dependencies" keyword is not defined' do
let(:options) { {} }
it { expect { job.run! }.not_to raise_error }
end
context 'when "dependencies" keyword is empty' do
let(:options) { { dependencies: [] } }
it { expect { job.run! }.not_to raise_error }
end
context 'when "dependencies" keyword is specified' do
let(:options) { { dependencies: ['test'] } }
it_behaves_like 'validation is active'
end
end
context 'when validates for dependencies is disabled' do
let(:options) { { dependencies: ['test'] } }
before do
stub_feature_flags(ci_disable_validates_dependencies: true)
end
it_behaves_like 'validation is not active'
end
end
describe 'state transition when build fails' do describe 'state transition when build fails' do
let(:service) { MergeRequests::AddTodoWhenBuildFailsService.new(project, user) } let(:service) { MergeRequests::AddTodoWhenBuildFailsService.new(project, user) }
......
...@@ -1244,7 +1244,7 @@ def create_pipeline(status, ref, sha, project) ...@@ -1244,7 +1244,7 @@ def create_pipeline(status, ref, sha, project)
describe '#execute_hooks' do describe '#execute_hooks' do
let!(:build_a) { create_build('a', 0) } let!(:build_a) { create_build('a', 0) }
let!(:build_b) { create_build('b', 1) } let!(:build_b) { create_build('b', 0) }
let!(:hook) do let!(:hook) do
create(:project_hook, project: project, pipeline_events: enabled) create(:project_hook, project: project, pipeline_events: enabled)
...@@ -1300,6 +1300,8 @@ def create_pipeline(status, ref, sha, project) ...@@ -1300,6 +1300,8 @@ def create_pipeline(status, ref, sha, project)
end end
context 'when stage one failed' do context 'when stage one failed' do
let!(:build_b) { create_build('b', 1) }
before do before do
build_a.drop build_a.drop
end end
......
...@@ -276,6 +276,89 @@ module Ci ...@@ -276,6 +276,89 @@ module Ci
end end
end end
context 'when "dependencies" keyword is specified' do
shared_examples 'not pick' do
it 'does not pick the build and drops the build' do
expect(subject).to be_nil
expect(pending_job.reload).to be_failed
expect(pending_job).to be_missing_dependency_failure
end
end
shared_examples 'validation is active' do
context 'when depended job has not been completed yet' do
let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) }
it_behaves_like 'not pick'
end
context 'when artifacts of depended job has been expired' do
let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
it_behaves_like 'not pick'
end
context 'when artifacts of depended job has been erased' do
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }
before do
pre_stage_job.erase
end
it_behaves_like 'not pick'
end
end
shared_examples 'validation is not active' do
context 'when depended job has not been completed yet' do
let!(:pre_stage_job) { create(:ci_build, :running, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect(subject).to eq(pending_job) }
end
context 'when artifacts of depended job has been expired' do
let!(:pre_stage_job) { create(:ci_build, :success, :expired, pipeline: pipeline, name: 'test', stage_idx: 0) }
it { expect(subject).to eq(pending_job) }
end
context 'when artifacts of depended job has been erased' do
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0, erased_at: 1.minute.ago) }
before do
pre_stage_job.erase
end
it { expect(subject).to eq(pending_job) }
end
end
before do
stub_feature_flags(ci_disable_validates_dependencies: false)
end
let!(:pre_stage_job) { create(:ci_build, :success, pipeline: pipeline, name: 'test', stage_idx: 0) }
let!(:pending_job) { create(:ci_build, :pending, pipeline: pipeline, stage_idx: 1, options: { dependencies: ['test'] } ) }
subject { execute(specific_runner) }
context 'when validates for dependencies is enabled' do
before do
stub_feature_flags(ci_disable_validates_dependencies: false)
end
it_behaves_like 'validation is active'
end
context 'when validates for dependencies is disabled' do
before do
stub_feature_flags(ci_disable_validates_dependencies: true)
end
it_behaves_like 'validation is not active'
end
end
def execute(runner) def execute(runner)
described_class.new(runner).execute.build described_class.new(runner).execute.build
end end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment